def s(data, dp, l_out, l1 ,l2, dp_type): # data = tf.nn.dropout(data, dp) # data = tf.expand_dims(data, axis=2) # data = tf.contrib.layers.batch_norm(data, activation_fn=tf.nn.relu) # data = tf.contrib.layers.conv2d(data, 1, 3, 1, 'SAME', activation_fn=None) # data = tf.contrib.layers.max_pool2d(data, 3, 2, 'SAME') if(dp_type == True): # l1 = tf.contrib.layers.batch_norm(data, activation_fn=tf.nn.relu, is_training=bn) layer1 = tf.contrib.layers.fully_connected(data, num_outputs=l_out, activation_fn=se.selu, weights_regularizer=tf.contrib.layers.l1_l2_regularizer(scale_l1=l1, scale_l2=l2), weights_initializer=tf.contrib.layers.variance_scaling_initializer(factor=1.0, mode='FAN_IN')) # l1 = tf.concat([data, l1], axis=1) layer1 = se.dropout_selu(layer1, rate=dp, training=(dp < 1.)) # l2 = tf.contrib.layers.batch_norm(l1, activation_fn=tf.nn.relu, is_training=bn) layer2 = tf.contrib.layers.fully_connected(layer1, num_outputs=l_out, activation_fn=se.selu, weights_regularizer=tf.contrib.layers.l1_l2_regularizer(scale_l1=l1, scale_l2=l2), weights_initializer=tf.contrib.layers.variance_scaling_initializer(factor=1.0, mode='FAN_IN')) # l2 = tf.concat([l1, l2], axis=1) layer2 = se.dropout_selu(layer2, rate=dp, training=(dp < 1.)) # l3 = tf.contrib.layers.batch_norm(l2, activation_fn=tf.nn.relu, is_training=bn) layer3 = tf.contrib.layers.fully_connected(layer2, num_outputs=l_out, activation_fn=se.selu, weights_regularizer=tf.contrib.layers.l1_l2_regularizer(scale_l1=l1, scale_l2=l2), weights_initializer=tf.contrib.layers.variance_scaling_initializer(factor=1.0, mode='FAN_IN')) else: # l1 = tf.contrib.layers.batch_norm(data, activation_fn=tf.nn.relu, is_training=bn) layer1 = tf.contrib.layers.fully_connected(data, num_outputs=l_out, activation_fn=se.selu, weights_regularizer=tf.contrib.layers.l1_l2_regularizer(scale_l1=l1, scale_l2=l2), weights_initializer=tf.contrib.layers.variance_scaling_initializer(factor=1.0, mode='FAN_IN')) # l1 = tf.concat([data, l1], axis=1) # l2 = tf.contrib.layers.batch_norm(l1, activation_fn=tf.nn.relu, is_training=bn) layer2 = tf.contrib.layers.fully_connected(tf.nn.dropout(layer1, dp), num_outputs=l_out, activation_fn=se.selu, weights_regularizer=tf.contrib.layers.l1_l2_regularizer(scale_l1=l1, scale_l2=l2), weights_initializer=tf.contrib.layers.variance_scaling_initializer(factor=1.0, mode='FAN_IN')) # l2 = tf.concat([l1, l2], axis=1) # l3 = tf.contrib.layers.batch_norm(l2, activation_fn=tf.nn.relu, is_training=bn) layer3 = tf.contrib.layers.fully_connected(tf.nn.dropout(layer2, dp), num_outputs=l_out, activation_fn=se.selu, weights_regularizer=tf.contrib.layers.l1_l2_regularizer(scale_l1=l1, scale_l2=l2), weights_initializer=tf.contrib.layers.variance_scaling_initializer(factor=1.0, mode='FAN_IN')) return layer3
def mlp_w_selu(x, weights, biases, dropout_rate, is_training): layer1 = tf.add(tf.matmul(x, weights['h1']), biases['b1']) layer1 = selu(layer1) layer1 = dropout_selu(layer1, dropout_rate, training=is_training) layer2 = tf.add(tf.matmul(layer1, weights['h2']), biases['b2']) layer2 = selu(layer2) layer2 = dropout_selu(layer2, dropout_rate, training=is_training) out_layer = tf.matmul(layer2, weights['out']) + biases['out'] # with linear activation return out_layer
def selu_adversary(x, z_pivot, n_layers, hidden_nodes, keep_prob, name, reuse=False, training=True, actv=selu.selu): SELU_initializer = tf.contrib.layers.variance_scaling_initializer( factor=1.0, mode='FAN_IN') xavier = tf.contrib.layers.xavier_initializer(uniform=True, seed=None, dtype=tf.float32) with tf.variable_scope('adversary', reuse=reuse): l0 = tf.layers.dense(x, units=hidden_nodes[0], activation=actv, kernel_initializer=SELU_initializer) d0 = selu.dropout_selu(l0, rate=1 - keep_prob, training=training) l1 = tf.layers.dense(d0, units=hidden_nodes[1], activation=actv, kernel_initializer=SELU_initializer) d1 = selu.dropout_selu(l1, rate=1 - keep_prob, training=training) fc = tf.layers.dense(d1, units=96, activation=tf.nn.tanh, kernel_initializer=xavier) fc_logits, fc_mu, fc_sigma = tf.split(fc, 3, axis=1) logits = tf.layers.dense(fc_logits, units=config.n_gaussians, activation=tf.identity, name='mixing_fractions') centers = tf.layers.dense(fc_mu, units=config.n_gaussians, activation=tf.identity, name='means') variances = tf.layers.dense(fc_sigma, units=config.n_gaussians, activation=custom_elu, name='variances') mixing_coeffs = tf.nn.softmax(logits) exponent = tf.log(mixing_coeffs) - 1 / 2 * tf.log(2 * np.pi) - tf.log( variances) - tf.square(centers - tf.expand_dims(z_pivot, 1)) / ( 2 * tf.square(variances)) return log_sum_exp_trick(exponent)
def dense_SELU(x, n_layers, hidden_layer_nodes, keep_prob, reuse=False, training=True, actv=selu.selu): SELU_initializer = tf.contrib.layers.variance_scaling_initializer( factor=1.0, mode='FAN_IN') with tf.variable_scope('seluNet', reuse=reuse): l0 = tf.layers.dense(x, units=hidden_layer_nodes[0], activation=actv, kernel_initializer=SELU_initializer) d0 = selu.dropout_selu(l0, rate=1 - keep_prob, training=training) l1 = tf.layers.dense(d0, units=hidden_layer_nodes[1], activation=actv, kernel_initializer=SELU_initializer) d1 = selu.dropout_selu(l1, rate=1 - keep_prob, training=training) l2 = tf.layers.dense(d1, units=hidden_layer_nodes[2], activation=actv, kernel_initializer=SELU_initializer) d2 = selu.dropout_selu(l2, rate=1 - keep_prob, training=training) l3 = tf.layers.dense(d2, units=hidden_layer_nodes[3], activation=actv, kernel_initializer=SELU_initializer) d3 = selu.dropout_selu(l3, rate=1 - keep_prob, training=training) l4 = tf.layers.dense(d3, units=hidden_layer_nodes[4], activation=actv, kernel_initializer=SELU_initializer) d4 = selu.dropout_selu(l4, rate=1 - keep_prob, training=training) # Readout layer readout = tf.layers.dense(d4, units=config.n_classes, kernel_initializer=SELU_initializer) return readout
def __call__(self, x, apply_softmax=True): del self.activations[:] xp = self.xp out = x for idx in xrange(self.num_layers): layer = getattr(self, "layer_%s" % idx) out = selu(layer(out)) if chainer.config.train == False: self.activations.append(xp.copy(out.data)) out = dropout_selu(out, ratio=0.1) out = self.logits(out) if apply_softmax: out = F.softmax(out) return out
def model(self, points_tensor, input_tensor, is_training=True): """ Arguments: points_tensor: [b, n, 3] point cloud input_tensor: [b, n, channels] extra data defined for each point """ b = points_tensor.get_shape()[0].value n = points_tensor.get_shape()[1].value in_channels = input_tensor.get_shape()[2].value voxel_size = tf.constant([0.1]) stride = tf.constant([1, 1, 1]) filter1_tensor = tf.get_variable("filter1", [3, 3, 3, in_channels, 9]) conv1 = conv3p(points_tensor, input_tensor, filter1_tensor, stride, voxel_size) relu1 = selu.selu(conv1) stride = tf.constant([2, 2, 2]) filter2_tensor = tf.get_variable("filter2", [3, 3, 3, 9, 9]) conv2 = conv3p(points_tensor, relu1, filter2_tensor, stride, voxel_size) relu2 = selu.selu(conv2) stride = tf.constant([3, 3, 3]) filter3_tensor = tf.get_variable("filter3", [3, 3, 3, 9, 9]) conv3 = conv3p(points_tensor, relu2, filter3_tensor, stride, voxel_size) relu3 = selu.selu(conv3) stride = tf.constant([4, 4, 4]) filter4_tensor = tf.get_variable("filter4", [3, 3, 3, 9, 9]) conv4 = conv3p(points_tensor, relu3, filter4_tensor, stride, voxel_size) relu4 = selu.selu(conv4) feat = tf.concat([relu1, relu2, relu3, relu4], axis=2) view = tf.reshape(feat, [-1, n * 36]) fc1 = tf.contrib.layers.fully_connected(view, 512, activation_fn=selu.selu) dropout = selu.dropout_selu(x=fc1, rate=0.5, training=is_training) fc2 = tf.contrib.layers.fully_connected(dropout, self.num_class, activation_fn=selu.selu) return fc2
def selu_builder(x, shape, name, keep_prob, training=True): init = tf.contrib.layers.variance_scaling_initializer(factor=1.0, mode='FAN_IN') with tf.variable_scope(name) as scope: W = tf.get_variable("weights", shape=shape, initializer=init) b = tf.get_variable( "biases", shape=[shape[1]], initializer=tf.random_normal_initializer(stddev=0.1)) actv = selu.selu(tf.add(tf.matmul(x, W), b)) layer_output = selu.dropout_selu(actv, rate=1 - keep_prob, training=training) return layer_output
def _buildGraph(self): with self.g.as_default(): XPH = tf.placeholder(tf.float32, [ None, self.inputShape[0], self.inputShape[1], self.inputShape[2] ], name='XPH') self.XPH = XPH YPH = tf.placeholder(tf.float32, [ None, self.outputShape1[0] + self.outputShape2[0] + self.outputShape3[0] + self.outputShape4[0] ], name='YPH') self.YPH = YPH learningRatePH = tf.placeholder(tf.float32, shape=[], name='learningRatePH') self.learningRatePH = learningRatePH phasePH = tf.placeholder(tf.bool, shape=[], name='phasePH') self.phasePH = phasePH dropoutRateFC4PH = tf.placeholder(tf.float32, shape=[], name='dropoutRateFC4PH') self.dropoutRateFC4PH = dropoutRateFC4PH dropoutRateFC5PH = tf.placeholder(tf.float32, shape=[], name='dropoutRateFC5PH') self.dropoutRateFC5PH = dropoutRateFC5PH l2RegularizationLambdaPH = tf.placeholder(tf.float32, shape=[], name='dropoutRateFC5PH') self.l2RegularizationLambdaPH = l2RegularizationLambdaPH conv1 = tf.layers.conv2d(inputs=XPH, filters=self.numFeature1, kernel_size=self.kernelSize1, kernel_initializer=tf.contrib.layers. variance_scaling_initializer( factor=2.0, mode='FAN_IN', uniform=False), padding="same", activation=selu.selu, name='conv1') self.conv1 = conv1 # print(conv1.shape) pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=self.pollSize1, strides=1, name='pool1') self.pool1 = pool1 # print(pool1.shape) conv2 = tf.layers.conv2d(inputs=pool1, filters=self.numFeature2, kernel_size=self.kernelSize2, kernel_initializer=tf.contrib.layers. variance_scaling_initializer( factor=2.0, mode='FAN_IN', uniform=False), padding="same", activation=selu.selu, name='conv2') self.conv2 = conv2 # print(conv2.shape) pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=self.pollSize2, strides=1, name='pool2') self.pool2 = pool2 # print(pool2.shape) conv3 = tf.layers.conv2d(inputs=pool2, filters=self.numFeature3, kernel_size=self.kernelSize3, kernel_initializer=tf.contrib.layers. variance_scaling_initializer( factor=2.0, mode='FAN_IN', uniform=False), padding="same", activation=selu.selu, name='conv3') self.conv3 = conv3 # print(conv3.shape) pool3 = tf.layers.max_pooling2d(inputs=conv3, pool_size=self.pollSize3, strides=1, name='pool3') self.pool3 = pool3 # print(pool3.shape) flat_size = (self.inputShape[0] - (self.pollSize1[0] - 1) - (self.pollSize2[0] - 1) - (self.pollSize3[0] - 1)) flat_size *= (self.inputShape[1] - (self.pollSize1[1] - 1) - (self.pollSize2[1] - 1) - (self.pollSize3[1] - 1)) flat_size *= self.numFeature3 conv3_flat = tf.reshape(pool3, [-1, flat_size]) # print(conv3_flat.shape) fc4 = tf.layers.dense(inputs=conv3_flat, units=self.hiddenLayerUnits4, kernel_initializer=tf.contrib.layers. variance_scaling_initializer(factor=2.0, mode='FAN_IN', uniform=False), activation=selu.selu, name='fc4') self.fc4 = fc4 # print(fc4.shape) dropout4 = selu.dropout_selu(fc4, dropoutRateFC4PH, training=phasePH, name='dropout4') self.dropout4 = dropout4 fc5 = tf.layers.dense(inputs=dropout4, units=self.hiddenLayerUnits5, kernel_initializer=tf.contrib.layers. variance_scaling_initializer(factor=2.0, mode='FAN_IN', uniform=False), activation=selu.selu, name='fc5') self.fc5 = fc5 # print(fc5.shape) dropout5 = selu.dropout_selu(fc5, dropoutRateFC5PH, training=phasePH, name='dropout5') self.dropout5 = dropout5 epsilon = tf.constant(value=1e-10) YBaseChangeSigmoid = tf.layers.dense(inputs=dropout4, units=self.outputShape1[0], activation=tf.nn.sigmoid, name='YBaseChangeSigmoid') self.YBaseChangeSigmoid = YBaseChangeSigmoid # print(YBaseChangeSigmoid.shape) YZygosityFC = tf.layers.dense(inputs=dropout5, units=self.outputShape2[0], activation=selu.selu, name='YZygosityFC') YZygosityLogits = tf.add(YZygosityFC, epsilon, name='YZygosityLogits') YZygositySoftmax = tf.nn.softmax(YZygosityLogits, name='YZygositySoftmax') self.YZygositySoftmax = YZygositySoftmax # print(YZygositySoftmax.shape) YVarTypeFC = tf.layers.dense(inputs=dropout5, units=self.outputShape3[0], activation=selu.selu, name='YVarTypeFC') YVarTypeLogits = tf.add(YVarTypeFC, epsilon, name='YVarTypeLogits') YVarTypeSoftmax = tf.nn.softmax(YVarTypeLogits, name='YVarTypeSoftmax') self.YVarTypeSoftmax = YVarTypeSoftmax # print(YVarTypeSoftmax.shape) YIndelLengthFC = tf.layers.dense(inputs=dropout5, units=self.outputShape4[0], activation=selu.selu, name='YIndelLengthFC') YIndelLengthLogits = tf.add(YIndelLengthFC, epsilon, name='YIndelLengthLogits') YIndelLengthSoftmax = tf.nn.softmax(YIndelLengthLogits, name='YIndelLengthSoftmax') self.YIndelLengthSoftmax = YIndelLengthSoftmax # print(YIndelLengthSoftmax.shape) # print(YPH.shape) #print(YBaseChangeSigmoid) #print(tf.slice(YPH,[0,0],[-1,self.outputShape1[0]])) loss1 = tf.reduce_sum( tf.pow(YBaseChangeSigmoid - tf.slice(YPH, [0, 0], [-1, self.outputShape1[0]], name='YBaseChangeGetTruth'), 2, name='YBaseChangeMSE'), name='YBaseChangeReduceSum') #print("Loss1: "+str(loss1)+"\n") #print(YZygosityLogits) #print(YZygosityCrossEntropy) #print(tf.slice(YPH, [0,self.outputShape1[0]], [-1,self.outputShape2[0]])) YZygosityCrossEntropy = tf.nn.log_softmax(YZygosityLogits, name='YZygosityLogSoftmax')\ * -tf.slice(YPH, [0,self.outputShape1[0]], [-1,self.outputShape2[0]], name='YZygosityGetTruth') loss2 = tf.reduce_sum(YZygosityCrossEntropy, name='YZygosityReduceSum') #print("Loss2: "+str(loss2)+"\n") #print(YVarTypeLogits) #print(YVarTypeCrossEntropy) #print(tf.slice(YPH, [0,self.outputShape1[0]+self.outputShape2[0]])) YVarTypeCrossEntropy = tf.nn.log_softmax(YVarTypeLogits, name='YVarTypeLogSoftmax')\ * -tf.slice(YPH, [0,self.outputShape1[0]+self.outputShape2[0]], [-1,self.outputShape3[0]], name='YVarTypeGetTruth') loss3 = tf.reduce_sum(YVarTypeCrossEntropy, name='YVarTypeReduceSum') #print("Loss3: " + str(loss3)+"\n") #print(YIndelLengthLogits) #print(YIndelLengthCrossEntropy) #print(tf.slice(YPH, [0,self.outputShape1[0]+self.outputShape2[0]+self.outputShape3[0]], [-1,self.outputShape4[0]])) YIndelLengthCrossEntropy = tf.nn.log_softmax(YIndelLengthLogits, name='YIndelLengthLogSoftmax')\ * -tf.slice(YPH, [0,self.outputShape1[0]+self.outputShape2[0]+self.outputShape3[0]], [-1,self.outputShape4[0]], name='YIndelLengthGetTruth') loss4 = tf.reduce_sum(YIndelLengthCrossEntropy, name='YIndelLengthReduceSum') #print("Loss4: " + str(loss4)+"\n") lossL2 = tf.add_n([ tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'bias' not in v.name ]) * l2RegularizationLambdaPH #print("LossL2: " + str(lossL2)+"\n") loss = loss1 + loss2 + loss3 + loss4 + lossL2 self.loss = loss # add tensorboard embedding self.embedding1 = YBaseChangeSigmoid self.embedding2 = YZygosityLogits self.embedding3 = YVarTypeLogits self.embedding4 = YIndelLengthLogits # add summaries tf.summary.scalar('learning_rate', learningRatePH) tf.summary.scalar('l2Lambda', l2RegularizationLambdaPH) tf.summary.scalar("loss1", loss1) tf.summary.scalar("loss2", loss2) tf.summary.scalar("loss3", loss3) tf.summary.scalar("loss4", loss4) tf.summary.scalar("lossL2", lossL2) tf.summary.scalar("loss", loss) # For report or debug. Fetching histogram summary is slow, GPU utilization will be low if enabled. #for var in tf.trainable_variables(): # tf.summary.histogram(var.op.name, var) self.merged_summary_op = tf.summary.merge_all() self.training_op = tf.train.AdamOptimizer( learning_rate=learningRatePH).minimize(loss) self.init_op = tf.global_variables_initializer()
#train = split[80:] #test = split[:20] split = int(0.8 * sol.shape[0]) #x_train = sol[:split, :] #y_train = sol_dt[:split, :] #x_test = sol[split:, :] #y_test = sol_dt[split:, :] #print("Training set", x_train.shape[0]) #print("Test set", x_test.shape[0]) model = Sequential() with tf.device('/gpu:0'): model.add( Dense(2048, kernel_initializer='glorot_uniform', input_shape=(42, ))) model.add(Lambda(lambda x: dropout_selu(x, 0.5))) for i in range(3): with tf.device(f'/gpu:{i+1}'): model.add(Dense(2048, kernel_initializer='he_uniform')) model.add(Lambda(lambda x: dropout_selu(x, 0.5))) model.add(Dense(41, activation='linear')) #model = make_parallel(model, 4) model.summary() model.compile(loss='mean_squared_error', optimizer=Adam(lr=0.01)) xtr = (sol[i:i + batch_size] for i in range(0, split, batch_size)) ytr = (sol_dt[i:i + batch_size] for i in range(0, split, batch_size))
def model(self, points_tensor, input_tensor, is_training=True): """ Arguments: points_tensor: [b, n, 3] point cloud input_tensor: [b, n, channels] extra data defined for each point """ b = points_tensor.get_shape()[0].value n = points_tensor.get_shape()[1].value in_channels = input_tensor.get_shape()[2].value voxel_size = tf.constant([0.1]) stride = tf.constant([1, 1, 1]) filter1_tensor = tf.get_variable("filter1", [3, 3, 3, in_channels, 9]) conv1 = conv3p(points_tensor, input_tensor, filter1_tensor, stride, voxel_size) relu1 = selu.selu(conv1) stride = tf.constant([2, 2, 2]) filter2_tensor = tf.get_variable("filter2", [3, 3, 3, 9, 9]) conv2 = conv3p(points_tensor, relu1, filter2_tensor, stride, voxel_size) relu2 = selu.selu(conv2) stride = tf.constant([3, 3, 3]) filter3_tensor = tf.get_variable("filter3", [3, 3, 3, 9, 9]) conv3 = conv3p(points_tensor, relu2, filter3_tensor, stride, voxel_size) relu3 = selu.selu(conv3) skipCon = tf.add(relu1, relu3, name=None) stride = tf.constant([4, 4, 4]) filter4_tensor = tf.get_variable("filter4", [3, 3, 3, 9, 9]) conv4 = conv3p(points_tensor, skipCon, filter4_tensor, stride, voxel_size) relu4 = selu.selu(conv4) stride = tf.constant([5, 5, 5]) filter5_tensor = tf.get_variable("filter5", [3, 3, 3, 9, 9]) conv5 = conv3p(points_tensor, relu4, filter5_tensor, stride, voxel_size) relu5 = selu.selu(conv5) skipCon2 = tf.add(relu3, relu5, name=None) stride = tf.constant([6, 6, 6]) filter6_tensor = tf.get_variable("filter6", [3, 3, 3, 9, 9]) conv6 = conv3p(points_tensor, skipCon2, filter6_tensor, stride, voxel_size) relu6 = selu.selu(conv6) stride = tf.constant([7, 7, 7]) filter7_tensor = tf.get_variable("filter7", [3, 3, 3, 9, 9]) conv7 = conv3p(points_tensor, relu6, filter7_tensor, stride, voxel_size) relu7 = selu.selu(conv7) skipCon3 = tf.add(relu5, relu7, name=None) stride = tf.constant([8, 8, 8]) filter8_tensor = tf.get_variable("filter8", [3, 3, 3, 9, 9]) conv8 = conv3p(points_tensor, skipCon3, filter8_tensor, stride, voxel_size) relu8 = selu.selu(conv8) feat = tf.concat( [relu1, relu2, skipCon, relu4, skipCon2, relu6, skipCon3, relu8], axis=2) #local features view = tf.reshape(feat, [-1, n * 72]) #global feature fc1 = tf.contrib.layers.fully_connected(view, 512, activation_fn=selu.selu) dropout = selu.dropout_selu(x=fc1, rate=0.5, training=is_training) fc2 = tf.contrib.layers.fully_connected(dropout, self.num_class, activation_fn=selu.selu) return fc2
def _buildGraph(self): with self.g.as_default(): XPH = tf.placeholder(tf.float32, [ None, self.inputShape[0], self.inputShape[1], self.inputShape[2] ], name='XPH') self.XPH = XPH YPH = tf.placeholder(tf.float32, [ None, self.outputShape1[0] + self.outputShape2[0] + self.outputShape3[0] + self.outputShape4[0] ], name='YPH') self.YPH = YPH learningRatePH = tf.placeholder(tf.float32, shape=[], name='learningRatePH') self.learningRatePH = learningRatePH phasePH = tf.placeholder(tf.bool, shape=[], name='phasePH') self.phasePH = phasePH dropoutRatePH = tf.placeholder(tf.float32, shape=[], name='dropoutRatePH') self.dropoutRatePH = dropoutRatePH conv1 = tf.layers.conv2d( inputs=XPH, filters=self.numFeature1, kernel_size=self.kernelSize1, kernel_initializer=tf.truncated_normal_initializer( stddev=1e-5, dtype=tf.float32), padding="same", activation=selu.selu, name='conv1') self.conv1 = conv1 pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=self.pollSize1, strides=1, name='pool1') self.pool1 = pool1 conv2 = tf.layers.conv2d( inputs=pool1, filters=self.numFeature2, kernel_size=self.kernelSize2, kernel_initializer=tf.truncated_normal_initializer( stddev=1e-5, dtype=tf.float32), padding="same", activation=selu.selu, name='conv2') self.conv2 = conv2 pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=self.pollSize2, strides=1, name='pool2') self.pool2 = pool2 conv3 = tf.layers.conv2d( inputs=pool2, filters=self.numFeature3, kernel_size=self.kernelSize3, kernel_initializer=tf.truncated_normal_initializer( stddev=1e-5, dtype=tf.float32), padding="same", activation=selu.selu, name='conv3') self.conv3 = conv3 pool3 = tf.layers.max_pooling2d(inputs=conv3, pool_size=self.pollSize3, strides=1, name='pool3') self.pool3 = pool3 flat_size = (self.inputShape[0] - (self.pollSize1[0] - 1) - (self.pollSize2[0] - 1) - (self.pollSize3[0] - 1)) flat_size *= (self.inputShape[1] - (self.pollSize1[1] - 1) - (self.pollSize2[1] - 1) - (self.pollSize3[1] - 1)) flat_size *= self.numFeature3 conv3_flat = tf.reshape(pool3, [-1, flat_size]) fc4 = tf.layers.dense( inputs=conv3_flat, units=self.hiddenLayerUnits4, kernel_initializer=tf.truncated_normal_initializer( stddev=1e-5, dtype=tf.float32), activation=selu.selu, name='fc4') self.fc4 = fc4 dropout4 = selu.dropout_selu(fc4, dropoutRatePH, training=phasePH, name='dropout4') self.dropout4 = dropout4 fc5 = tf.layers.dense( inputs=dropout4, units=self.hiddenLayerUnits5, kernel_initializer=tf.truncated_normal_initializer( stddev=1e-5, dtype=tf.float32), activation=selu.selu, name='fc5') self.fc5 = fc5 dropout5 = selu.dropout_selu(fc5, dropoutRatePH, training=phasePH, name='dropout5') self.dropout5 = dropout5 epsilon = tf.constant(value=1e-10) YBaseChangeSigmoid = tf.layers.dense(inputs=dropout5, units=self.outputShape1[0], activation=tf.nn.sigmoid, name='YBaseChangeSigmoid') self.YBaseChangeSigmoid = YBaseChangeSigmoid YZygosityFC = tf.layers.dense(inputs=dropout5, units=self.outputShape2[0], activation=selu.selu, name='YZygosityFC') YZygosityLogits = tf.add(YZygosityFC, epsilon, name='YZygosityLogits') YZygositySoftmax = tf.nn.softmax(YZygosityLogits, name='YZygositySoftmax') self.YZygositySoftmax = YZygositySoftmax YVarTypeFC = tf.layers.dense(inputs=dropout5, units=self.outputShape3[0], activation=selu.selu, name='YVarTypeFC') YVarTypeLogits = tf.add(YVarTypeFC, epsilon, name='YVarTypeLogits') YVarTypeSoftmax = tf.nn.softmax(YVarTypeLogits, name='YVarTypeSoftmax') self.YVarTypeSoftmax = YVarTypeSoftmax YIndelLengthFC = tf.layers.dense(inputs=dropout5, units=self.outputShape4[0], activation=selu.selu, name='YIndelLengthFC') YIndelLengthLogits = tf.add(YIndelLengthFC, epsilon, name='YIndelLengthLogits') YIndelLengthSoftmax = tf.nn.softmax(YIndelLengthLogits, name='YIndelLengthSoftmax') self.YIndelLengthSoftmax = YIndelLengthSoftmax loss1 = tf.reduce_sum( tf.pow(YBaseChangeSigmoid - tf.slice(YPH, [0, 0], [-1, self.outputShape1[0]], name='YBaseChangeGetTruth'), 2, name='YBaseChangeMSE'), name='YBaseChangeReduceSum') YZygosityCrossEntropy = tf.nn.log_softmax(YZygosityLogits, name='YZygosityLogSoftmax')\ * -tf.slice(YPH, [0,self.outputShape1[0]], [-1,self.outputShape2[0]], name='YZygosityGetTruth') loss2 = tf.reduce_sum(YZygosityCrossEntropy, name='YZygosityReduceSum') YVarTypeCrossEntropy = tf.nn.log_softmax(YVarTypeLogits, name='YVarTypeLogSoftmax')\ * -tf.slice(YPH, [0,self.outputShape1[0]+self.outputShape2[0]], [-1,self.outputShape3[0]], name='YVarTypeGetTruth') loss3 = tf.reduce_sum(YVarTypeCrossEntropy, name='YVarTypeReduceSum') YIndelLengthCrossEntropy = tf.nn.log_softmax(YIndelLengthLogits, name='YIndelLengthLogSoftmax')\ * -tf.slice(YPH, [0,self.outputShape1[0]+self.outputShape2[0]+self.outputShape3[0]], [-1,self.outputShape4[0]], name='YIndelLengthGetTruth') loss4 = tf.reduce_sum(YIndelLengthCrossEntropy, name='YIndelLengthReduceSum') loss = loss1 + loss2 + loss3 + loss4 self.loss = loss # add summaries tf.summary.scalar('learning_rate', learningRatePH) tf.summary.scalar("loss1", loss1) tf.summary.scalar("loss2", loss2) tf.summary.scalar("loss3", loss3) tf.summary.scalar("loss4", loss4) tf.summary.scalar("loss", loss) # For report or debug. Fetching histogram summary is slow, GPU utilization will be low if enabled. #for var in tf.trainable_variables(): # tf.summary.histogram(var.op.name, var) self.merged_summary_op = tf.summary.merge_all() self.training_op = tf.train.AdamOptimizer( learning_rate=learningRatePH).minimize(loss) self.init_op = tf.global_variables_initializer()
def _buildGraph(self): with self.g.as_default(): XPH = tf.placeholder(tf.float32, [ None, self.inputShape[0], self.inputShape[1], self.inputShape[2] ], name='XPH') self.XPH = XPH YPH = tf.placeholder(tf.float32, [ None, self.outputShape1[0] + self.outputShape2[0] + self.outputShape3[0] + self.outputShape4[0] ], name='YPH') self.YPH = YPH learningRatePH = tf.placeholder(tf.float32, shape=[], name='learningRatePH') self.learningRatePH = learningRatePH phasePH = tf.placeholder(tf.bool, shape=[], name='phasePH') self.phasePH = phasePH dropoutRatePH = tf.placeholder(tf.float32, shape=[], name='dropoutRatePH') self.dropoutRatePH = dropoutRatePH conv1 = tf.layers.conv2d( inputs=XPH, filters=self.numFeature1, kernel_size=self.kernelSize1, kernel_initializer=tf.truncated_normal_initializer( stddev=1e-5, dtype=tf.float32), padding="same", activation=selu.selu, name='conv1') self.conv1 = conv1 conv2 = tf.layers.conv2d( inputs=conv1, filters=self.numFeature2, kernel_size=self.kernelSize2, kernel_initializer=tf.truncated_normal_initializer( stddev=1e-5, dtype=tf.float32), padding="same", activation=selu.selu, name='conv2') self.conv2 = conv2 conv3 = tf.layers.conv2d( inputs=conv2, filters=self.numFeature3, kernel_size=self.kernelSize3, kernel_initializer=tf.truncated_normal_initializer( stddev=1e-5, dtype=tf.float32), padding="same", activation=selu.selu, name='conv3') self.conv3 = conv3 flat_size = self.inputShape[0] * self.inputShape[ 1] * self.numFeature3 conv3_flat = tf.reshape(conv3, [-1, flat_size]) fc4 = tf.layers.dense( inputs=conv3_flat, units=self.hiddenLayerUnits4, kernel_initializer=tf.truncated_normal_initializer( stddev=1e-5, dtype=tf.float32), activation=selu.selu, name='fc4') self.fc4 = fc4 dropout4 = selu.dropout_selu(fc4, dropoutRatePH, training=phasePH, name='dropout4') self.dropout4 = dropout4 fc5 = tf.layers.dense( inputs=dropout4, units=self.hiddenLayerUnits5, kernel_initializer=tf.truncated_normal_initializer( stddev=1e-5, dtype=tf.float32), activation=selu.selu, name='fc5') self.fc5 = fc5 dropout5 = selu.dropout_selu(fc5, dropoutRatePH, training=phasePH, name='dropout5') self.dropout5 = dropout5 epsilon = tf.constant(value=1e-10) YBaseChangeSigmoid = tf.layers.dense(inputs=dropout5, units=self.outputShape1[0], activation=tf.nn.sigmoid, name='YBaseChangeSigmoid') self.YBaseChangeSigmoid = YBaseChangeSigmoid YZygosityFC = tf.layers.dense(inputs=dropout5, units=self.outputShape2[0], activation=selu.selu, name='YZygosityFC') YZygosityLogits = YZygosityFC + epsilon YZygositySoftmax = tf.nn.softmax(YZygosityLogits, name='YZygositySoftmax') self.YZygositySoftmax = YZygositySoftmax YVarTypeFC = tf.layers.dense(inputs=dropout5, units=self.outputShape3[0], activation=selu.selu, name='YVarTypeFC') YVarTypeLogits = YVarTypeFC + epsilon YVarTypeSoftmax = tf.nn.softmax(YVarTypeLogits, name='YVarTypeSoftmax') self.YVarTypeSoftmax = YVarTypeSoftmax YIndelLengthFC = tf.layers.dense(inputs=dropout5, units=self.outputShape4[0], activation=selu.selu, name='YIndelLengthFC') YIndelLengthLogits = YIndelLengthFC + epsilon YIndelLengthSoftmax = tf.nn.softmax(YIndelLengthLogits, name='YIndelLengthSoftmax') self.YIndelLengthSoftmax = YIndelLengthSoftmax loss1 = tf.reduce_sum( tf.pow( YBaseChangeSigmoid - tf.slice(YPH, [0, 0], [-1, self.outputShape1[0]]), 2)) YZygosityCrossEntropy = tf.nn.log_softmax(YZygosityLogits)\ * -tf.slice(YPH, [0,self.outputShape1[0]], [-1,self.outputShape2[0]]) loss2 = tf.reduce_sum(YZygosityCrossEntropy) YVarTypeCrossEntropy = tf.nn.log_softmax(YVarTypeLogits)\ * -tf.slice(YPH, [0,self.outputShape1[0]+self.outputShape2[0]], [-1,self.outputShape3[0]]) loss3 = tf.reduce_sum(YVarTypeCrossEntropy) YIndelLengthCrossEntropy = tf.nn.log_softmax(YIndelLengthLogits)\ * -tf.slice(YPH, [0,self.outputShape1[0]+self.outputShape2[0]+self.outputShape3[0]], [-1,self.outputShape4[0]]) loss4 = tf.reduce_sum(YIndelLengthCrossEntropy) loss = loss1 + loss2 + loss3 + loss4 self.loss = loss # add summaries tf.summary.scalar('learning_rate', learningRatePH) tf.summary.scalar("loss1", loss1) tf.summary.scalar("loss2", loss2) tf.summary.scalar("loss3", loss3) tf.summary.scalar("loss4", loss4) tf.summary.scalar("loss", loss) #for var in tf.trainable_variables(): # tf.summary.histogram(var.op.name, var) self.merged_summary_op = tf.summary.merge_all() self.training_op = tf.train.AdamOptimizer( learning_rate=learningRatePH).minimize(loss) self.init_op = tf.global_variables_initializer()
def model(self, points_tensor, input_tensor, is_training=True): """ Arguments: points_tensor: [b, n, 3] point cloud input_tensor: [b, n, channels] extra data defined for each point """ b = points_tensor.get_shape()[0].value n = points_tensor.get_shape()[1].value in_channels = input_tensor.get_shape()[2].value voxel_size = tf.constant([0.1]) stride = tf.constant([1, 1, 1]) filter1_tensor = tf.get_variable("filter1", [3, 3, 3, in_channels, 9]) conv1 = conv3p(points_tensor, input_tensor, filter1_tensor, stride, voxel_size); relu1 = selu.selu(conv1) stride = tf.constant([2, 2, 2]) filter2_tensor = tf.get_variable("filter2", [3, 3, 3, 9, 9]) conv2 = conv3p(points_tensor, relu1, filter2_tensor, stride, voxel_size); relu2 = selu.selu(conv2) stride = tf.constant([3, 3, 3]) filter3_tensor = tf.get_variable("filter3", [3, 3, 3, 9, 9]) conv3 = conv3p(points_tensor, relu2, filter3_tensor, stride, voxel_size); skipCon1 = tf.add(relu1, conv3) relu3 = selu.selu(skipCon1) stride = tf.constant([4, 4, 4]) filter4_tensor = tf.get_variable("filter4", [3, 3, 3, 9, 9]) conv4 = conv3p(points_tensor, relu3, filter4_tensor, stride, voxel_size); relu4 = selu.selu(conv4) stride = tf.constant([5, 5, 5]) filter5_tensor = tf.get_variable("filter5", [3, 3, 3, 9, 9]) conv5 = conv3p(points_tensor, relu4, filter5_tensor, stride, voxel_size); skipCon2 = tf.add(relu3, conv5) relu5 = selu.selu(skipCon2) stride = tf.constant([6, 6, 6]) filter6_tensor = tf.get_variable("filter6", [3, 3, 3, 9, 9]) conv6 = conv3p(points_tensor, relu5, filter6_tensor, stride, voxel_size); relu6 = selu.selu(conv6) stride = tf.constant([7, 7, 7]) filter7_tensor = tf.get_variable("filter7", [3, 3, 3, 9, 9]) conv7 = conv3p(points_tensor, relu6, filter7_tensor, stride, voxel_size); skipCon3 = tf.add(relu5, conv7, name = None) relu7 = selu.selu(skipCon3) stride = tf.constant([8, 8, 8]) filter8_tensor = tf.get_variable("filter8", [3, 3, 3, 9, 9]) conv8 = conv3p(points_tensor, relu7, filter8_tensor, stride, voxel_size); relu8 = selu.selu(conv8) stride = tf.constant([9, 9, 9]) filter9_tensor = tf.get_variable("filter9", [3, 3, 3, 9, 9]) conv9 = conv3p(points_tensor, relu8, filter9_tensor, stride, voxel_size); skipCon4 = tf.add(relu7, conv9) relu9 = selu.selu(skipCon4) stride = tf.constant([10, 10, 10]) filter10_tensor = tf.get_variable("filter10", [3, 3, 3, 9, 9]) conv10 = conv3p(points_tensor, relu9, filter10_tensor, stride, voxel_size); relu10 = selu.selu(conv10) stride = tf.constant([11, 11, 11]) filter11_tensor = tf.get_variable("filter11", [3, 3, 3, 9, 9]) conv11 = conv3p(points_tensor, relu10, filter11_tensor, stride, voxel_size); skipCon5 = tf.add(relu9, conv11) relu11 = selu.selu(skipCon5) stride = tf.constant([12, 12, 12]) filter12_tensor = tf.get_variable("filter12", [3, 3, 3, 9, 9]) conv12 = conv3p(points_tensor, relu11, filter12_tensor, stride, voxel_size); relu12 = selu.selu(conv12) stride = tf.constant([13, 13, 13]) filter13_tensor = tf.get_variable("filter13", [3, 3, 3, 9, 9]) conv13 = conv3p(points_tensor, relu12, filter13_tensor, stride, voxel_size); skipCon6 = tf.add(relu11, conv13,name = None) relu13 = selu.selu(skipCon6) stride = tf.constant([14, 14, 14]) filter14_tensor = tf.get_variable("filter14", [3, 3, 3, 9, 9]) conv14 = conv3p(points_tensor, relu13, filter14_tensor, stride, voxel_size); relu14 = selu.selu(conv14) stride = tf.constant([15, 15, 15]) filter15_tensor = tf.get_variable("filter15", [3, 3, 3, 9, 9]) conv15 = conv3p(points_tensor, relu14, filter15_tensor, stride, voxel_size); skipCon7 = tf.add(relu13, conv15) relu15 = selu.selu(skipCon7) stride = tf.constant([16, 16, 16]) filter16_tensor = tf.get_variable("filter16", [3, 3, 3, 9, 9]) conv16 = conv3p(points_tensor, relu15, filter16_tensor, stride, voxel_size); relu16 = selu.selu(conv16) feat = tf.concat([relu1, relu2, relu3, relu4, relu5, relu6, relu7, relu8, relu9, relu10, relu11, relu12, relu13, relu14, relu15, relu16], axis=2) #local features view = tf.reshape(feat, [-1, n * 144]) #global feature fc1 = tf.contrib.layers.fully_connected(view, 512 , activation_fn=selu.selu) dropout1 = selu.dropout_selu(x=fc1, rate=0.5, training=is_training) fc2 = tf.contrib.layers.fully_connected(dropout1, self.num_class, activation_fn=selu.selu) return fc2
def _build_graph(self): """ Build the computation graph for the model """ self.graph = self.g self.layers = [ ] # A list used to contain meaningful intermediate layers with self.graph.as_default(): tf.set_random_seed(param.RANDOM_SEED) # Conversion to tensors for some values self.epsilon = tf.constant(value=1e-10, dtype=self.float_type) self.input_shape_tf = (None, self.input_shape[0], self.input_shape[1], self.input_shape[2]) # Place holders self.X_placeholder = tf.placeholder(self.float_type, self.input_shape_tf, name='X_placeholder') self.Y_placeholder = tf.placeholder(self.float_type, [ None, self.output_base_change_shape + self.output_zygosity_shape + self.output_variant_type_shape + self.output_indel_length_shape ], name='Y_placeholder') self.layers.append(self.X_placeholder) self.learning_rate_placeholder = tf.placeholder( self.float_type, shape=[], name='learning_rate_placeholder') self.phase_placeholder = tf.placeholder(tf.bool, shape=[], name='phase_placeholder') self.regularization_L2_lambda_placeholder = tf.placeholder( self.float_type, shape=[], name='regularization_L2_lambda_placeholder') self.task_loss_weights_placeholder = tf.placeholder( self.float_type, shape=self.task_loss_weights.shape, name='task_loss_weights_placeholder') self.output_base_change_entropy_weights_placeholder = tf.placeholder( self.float_type, shape=self.output_base_change_entropy_weights.shape, name='output_base_change_entropy_weights_placeholder') self.output_zygosity_entropy_weights_placeholder = tf.placeholder( self.float_type, shape=self.output_zygosity_entropy_weights.shape, name='output_zygosity_entropy_weights_placeholder') self.output_variant_type_entropy_weights_placeholder = tf.placeholder( self.float_type, shape=self.output_variant_type_entropy_weights.shape, name='output_variant_type_entropy_weights_placeholder') self.output_indel_length_entropy_weights_placeholder = tf.placeholder( self.float_type, shape=self.output_indel_length_entropy_weights.shape, name='output_indel_length_entropy_weights_placeholder') he_initializer = tf.contrib.layers.variance_scaling_initializer( factor=1.0, mode='FAN_IN', seed=param.OPERATION_SEED) if self.structure == "2BiLSTM": self.L3_dropout_rate_placeholder = tf.placeholder( self.float_type, shape=[], name='L3_dropout_rate_placeholder') # Flatten the 2nd (ACGT) and 3rd (Ref Ins Del SNP) dimension self.X_flattened_2D = tf.reshape( self.X_placeholder, shape=(tf.shape(self.X_placeholder)[0], self.input_shape_tf[1], self.input_shape_tf[2] * self.input_shape_tf[3]), name="X_flattened_2D") self.layers.append(self.X_flattened_2D) self.X_flattened_2D_transposed = tf.transpose( self.X_flattened_2D, [1, 0, 2], name="X_flattened_2D_transposed") is_gpu_available = len(Clair.get_available_gpus()) > 0 # print("is_gpu_available:", is_gpu_available) self.LSTM1, self.LSTM1_state = Clair.adaptive_LSTM_layer( inputs=self.X_flattened_2D_transposed, num_units=self.LSTM1_num_units, name="LSTM1", direction="bidirectional", num_layers=1, cudnn_gpu_available=is_gpu_available) self.layers.append(self.LSTM1) # print(self.LSTM1, self.LSTM1_state) self.LSTM1_dropout = tf.layers.dropout( inputs=self.LSTM1, rate=self.LSTM1_dropout_rate, training=self.phase_placeholder, name="LSTM1_dropout", seed=param.OPERATION_SEED) self.LSTM2, _ = Clair.adaptive_LSTM_layer( inputs=self.LSTM1_dropout, num_units=self.LSTM2_num_units, name="LSTM2", direction="bidirectional", num_layers=1, cudnn_gpu_available=is_gpu_available) self.layers.append(self.LSTM2) self.LSTM2_dropout = tf.layers.dropout( inputs=self.LSTM2, rate=self.LSTM2_dropout_rate, training=self.phase_placeholder, name="LSTM2_dropout", seed=param.OPERATION_SEED) self.LSTM2_transposed = tf.transpose(self.LSTM2_dropout, [1, 0, 2], name="LSTM2_transposed") # Slice dense layer 2 self.L2 = Clair.slice_dense_layer( inputs=self.LSTM2_transposed, units=self.L2_num_units, slice_dimension=2, name="L2", activation=selu.selu, kernel_initializer=he_initializer) self.layers.append(self.L2) self.L2_flattened = tf.reshape( self.L2, shape=(tf.shape(self.L2)[0], self.L2_num_units * self.LSTM2_num_units * 2), name="L2_flattened") self.layers.append(self.L2_flattened) # Dense layer 3 self.L3 = tf.layers.dense(inputs=self.L2_flattened, units=self.L3_num_units, name="L3", activation=selu.selu, kernel_initializer=he_initializer) self.layers.append(self.L3) self.L3_dropout = selu.dropout_selu( self.L3, self.L3_dropout_rate_placeholder, training=self.phase_placeholder, name='L3_dropout', seed=param.OPERATION_SEED) self.layers.append(self.L3_dropout) self.core_final_layer = self.L3_dropout # Output layer with tf.variable_scope("Prediction"): self.Y_base_change_logits = tf.layers.dense( inputs=self.core_final_layer, units=self.output_base_change_shape, kernel_initializer=he_initializer, activation=selu.selu, name='Y_base_change_logits') self.Y_base_change = tf.nn.softmax(self.Y_base_change_logits, name='Y_base_change') self.layers.append(self.Y_base_change) self.Y_zygosity_logits = tf.layers.dense( inputs=self.core_final_layer, units=self.output_zygosity_shape, kernel_initializer=he_initializer, activation=selu.selu, name='Y_zygosity_logits') self.Y_zygosity = tf.nn.softmax(self.Y_zygosity_logits, name='Y_zygosity') self.layers.append(self.Y_zygosity) if "legacy_0.1" in self.structure: self.Y_variant_type_logits = tf.layers.dense( inputs=self.core_final_layer, units=self.output_variant_type_shape, kernel_initializer=he_initializer, activation=selu.selu, name='Y_variant_logits') self.Y_variant_type = tf.nn.softmax( self.Y_variant_type_logits, name='Y_variant') else: self.Y_variant_type_logits = tf.layers.dense( inputs=self.core_final_layer, units=self.output_variant_type_shape, kernel_initializer=he_initializer, activation=selu.selu, name='Y_variant_type_logits') self.Y_variant_type = tf.nn.softmax( self.Y_variant_type_logits, name='Y_variant_type') self.layers.append(self.Y_variant_type) self.Y_indel_length_logits = tf.layers.dense( inputs=self.core_final_layer, units=self.output_indel_length_shape, kernel_initializer=he_initializer, activation=selu.selu, name='Y_indel_length_logits') self.Y_indel_length = tf.nn.softmax(self.Y_indel_length_logits, name='Y_indel_length') self.layers.append(self.Y_indel_length) self.Y = [ self.Y_base_change, self.Y_zygosity, self.Y_variant_type, self.Y_indel_length ] # Extract the truth labels by output ratios with tf.variable_scope("Loss"): Y_base_change_label, Y_zygosity_label, Y_variant_type_label, Y_indel_length_label = tf.split( self.Y_placeholder, self.output_label_split, axis=1, name="label_split") # Cross Entropy loss Y_variant_type_str = "Y_variant_type" self.Y_base_change_cross_entropy = Clair.weighted_cross_entropy( softmax_prediction=self.Y_base_change, labels=Y_base_change_label, weights=self. output_base_change_entropy_weights_placeholder, epsilon=self.epsilon, name="Y_base_change_cross_entropy") self.Y_zygosity_cross_entropy = Clair.weighted_cross_entropy( softmax_prediction=self.Y_zygosity, labels=Y_zygosity_label, weights=self.output_zygosity_entropy_weights_placeholder, epsilon=self.epsilon, name="Y_zygosity_cross_entropy") self.Y_variant_type_cross_entropy = Clair.weighted_cross_entropy( softmax_prediction=self.Y_variant_type, labels=Y_variant_type_label, weights=self. output_variant_type_entropy_weights_placeholder, epsilon=self.epsilon, name=Y_variant_type_str + "_cross_entropy") self.Y_indel_length_entropy = Clair.weighted_cross_entropy( softmax_prediction=self.Y_indel_length, labels=Y_indel_length_label, weights=self. output_indel_length_entropy_weights_placeholder, epsilon=self.epsilon, name="Y_indel_length_entropy") self.Y_base_change_loss = tf.reduce_sum( self.Y_base_change_cross_entropy, name="Y_base_change_loss") self.Y_zygosity_loss = tf.reduce_sum( self.Y_zygosity_cross_entropy, name="Y_zygosity_loss") self.Y_variant_type_loss = tf.reduce_sum( self.Y_variant_type_cross_entropy, name=Y_variant_type_str + "_loss") self.Y_indel_length_loss = tf.reduce_sum( self.Y_indel_length_entropy, name="Y_indel_length_loss") self.regularization_L2_loss_without_lambda = tf.add_n([ tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'bias' not in v.name ]) self.regularization_L2_loss = ( self.regularization_L2_loss_without_lambda * self.regularization_L2_lambda_placeholder) # Weighted average of losses, speicified by loss_ratio self.total_loss = tf.reduce_sum(tf.multiply( self.task_loss_weights_placeholder, tf.stack([ self.Y_base_change_loss, self.Y_zygosity_loss, self.Y_variant_type_loss, self.Y_indel_length_loss, self.regularization_L2_loss ])), name="Total_loss") # Create the saver for the model self.saver = tf.train.Saver(max_to_keep=1000000, ) # Include gradient clipping if RNN architectures are used if "RNN" in self.structure or "LSTM" in self.structure: with tf.variable_scope("Training_Operation"): self.optimizer = tf.train.AdamOptimizer( learning_rate=self.learning_rate_placeholder) gradients, variables = zip( *self.optimizer.compute_gradients(self.total_loss)) gradients, _ = tf.clip_by_global_norm(gradients, 5.0) self.training_op = self.optimizer.apply_gradients( zip(gradients, variables)) else: self.training_op = tf.train.AdamOptimizer( learning_rate=self.learning_rate_placeholder).minimize( self.total_loss) self.init_op = tf.global_variables_initializer() # Summary logging self.training_summary_op = tf.summary.merge([ tf.summary.scalar('learning_rate', self.learning_rate_placeholder), tf.summary.scalar('l2_Lambda', self.regularization_L2_lambda_placeholder), tf.summary.scalar("Y_base_change_loss", self.Y_base_change_loss), tf.summary.scalar("Y_zygosity_loss", self.Y_zygosity_loss), tf.summary.scalar("Y_variant_type_loss", self.Y_variant_type_loss), tf.summary.scalar("Y_indel_length_loss", self.Y_indel_length_loss), tf.summary.scalar("Regularization_loss", self.regularization_L2_loss), tf.summary.scalar("Total_loss", self.total_loss) ]) # For report or debug. Fetching histogram summary is slow, GPU utilization will be low if enabled. # for var in tf.trainable_variables(): # tf.summary.histogram(var.op.name, var) # self.merged_summary_op = tf.summary.merge_all() # Aliasing self.loss = self.total_loss