def create_conv(prev, filter_size, nb): w_filters = tf.Variable( tf.truncated_normal(shape=(filter_size, filter_size, int(prev.get_shape()[-1]), nb))) b_filters = tf.Variable(tf.zeros(shape=(nb))) conv = tf.nn.conv2d(prev, w_filters, strides=[1, 1, 1, 1], padding="SAME") + b_filters conv = tf.nn.relu(conv) conv = tf.nn.max_pool(conv, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="SAME") return conv
def init_weights(shape): ''' Initialize weight of a layer using a normal distribution Args: - shape: shape of the weights Returns: - W (tf.Variable): initialized weights of shape `input_shape` ''' init_weight = tf.truncated_normal(shape=shape, stddev=0.1) W = tf.Variable(init_weight) return W
def buildGraph(loss="MSE", beta1=0.9, beta2=0.999, epsilon=1e-08): W = tf.Variable(tf.truncated_normal(shape=[28 * 28, 1], stddev=0.5, name="weights")) b = tf.Variable(0.0, name="biases") x = tf.placeholder(tf.float32, shape=[None, 28 * 28]) y = tf.placeholder(tf.float32, shape=[None, 1]) validData = tf.placeholder(tf.float32, shape=[None, 28 * 28]) validTarget = tf.placeholder(tf.float32, shape=[None, 1]) testData = tf.placeholder(tf.float32, shape=[None, 28 * 28]) testTarget = tf.placeholder(tf.float32, shape=[None, 1]) reg = tf.placeholder(tf.float32) tf.set_random_seed(421) if loss == "MSE": y_pred = tf.matmul(x, W) + b valid_pred = tf.matmul(validData, W) + b test_pred = tf.matmul(testData, W) + b train_loss = tf.losses.mean_squared_error(labels=y, predictions=y_pred) + \ reg * tf.nn.l2_loss(W) optimizer = tf.train.AdamOptimizer(learning_rate=0.001, beta1=beta1, \ beta2=beta2, epsilon=epsilon) train = optimizer.minimize(loss=train_loss) valid_loss = tf.losses.mean_squared_error(labels=validTarget, \ redictions=valid_pred) + reg * tf.nn.l2_loss(W) test_loss = tf.losses.mean_squared_error(labels=testTarget, \ predictions=test_pred) + reg * tf.nn.l2_loss(W) elif loss == "CE": train_logits = tf.matmul(x, W) + b valid_logits = tf.matmul(validData, W) + b test_logits = tf.matmul(testData, W) + b train_loss = tf.losses.sigmoid_cross_entropy(y, train_logits) + \ reg * tf.nn.l2_loss(W) optimizer = tf.train.AdamOptimizer(learning_rate=0.001, \ beta1=beta1, beta2=beta2, epsilon=epsilon) train = optimizer.minimize(loss=train_loss) valid_loss = tf.losses.sigmoid_cross_entropy(validTarget, valid_logits) + \ reg * tf.nn.l2_loss(W) test_loss = tf.losses.sigmoid_cross_entropy(testTarget, test_logits) + \ reg * tf.nn.l2_loss(W) y_pred = tf.sigmoid(train_logits) valid_pred = tf.sigmoid(valid_logits) test_pred = tf.sigmoid(test_logits) return W, b, x, y_pred, y, validData, validTarget, testData, testTarget, reg, \ train_loss, valid_loss, test_loss, train, valid_pred, test_pred
def upsample_and_concat(x1, x2, output_channels, in_channels): pool_size = 2 deconv_filter = tf.Variable( tf.truncated_normal( [pool_size, pool_size, output_channels, in_channels], stddev=0.02)) deconv = tf.nn.conv2d_transpose(x1, deconv_filter, tf.shape(x2), strides=[1, pool_size, pool_size, 1]) deconv_output = tf.concat([deconv, x2], 3) deconv_output.set_shape([None, None, None, output_channels * 2]) return deconv_output
def kaiming(shape, dtype, partition_info=None): """Kaiming initialization as described in https://arxiv.org/pdf/1502.01852.pdf Args shape: dimensions of the tf array to initialize dtype: data type of the array partition_info: (Optional) info about how the variable is partitioned. See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/init_ops.py#L26 Needed to be used as an initializer. Returns Tensorflow array with initial weights """ return (tf.truncated_normal(shape, dtype=dtype) * tf.sqrt(2 / float(shape[0])))
def __init__(self, vehicles_train_path, non_vehicles_train_path, vehicles_test_path, non_vehicles_test_path, image_shape: tuple, batch_size: int, train_classes_ratio: Fraction, train_rate: float, train_num_iterate): self.__batch_size__ = batch_size self.__train_classes_ratio__ = train_classes_ratio self.__images_cntrl__ = ImagesController(vehicles_train_path, non_vehicles_train_path, vehicles_test_path, non_vehicles_test_path, self.__batch_size__, self.__train_classes_ratio__) self.__image_shape__ = image_shape self.__train_num_iterate__ = train_num_iterate self.__train_rate__ = train_rate self.__num_inputs__ = image_shape[0] * image_shape[1] self.__input__ = tf.placeholder(tf.float32, [None, self.__num_inputs__]) self.__weights__ = tf.Variable(tf.truncated_normal(shape=[self.__num_inputs__, 1], stddev=0.1)) self.__bias__ = tf.Variable(tf.truncated_normal(shape=[1], stddev=0.1)) self.__output__ = tf.nn.sigmoid(tf.matmul(self.__input__, self.__weights__) + self.__bias__) self.__real__ = tf.placeholder(tf.float32, [None, 1]) self.__loss__ = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=self.__real__, logits=self.__output__)) self.__update__ = tf.train.GradientDescentOptimizer(self.__train_rate__).minimize(self.__loss__) self.__sess__ = tf.Session() self.__initialized__ = False self.__closed__ = False
def _init_conv_layers(self): self.__conv_weights__ = [] self.__conv_biases__ = [] self.__conv_outputs__ = [] self.__conv_activations__ = [] self.__conv_pools__ = [] kernel_r = self.__conv_kernel_dimensions__[ 0] # First dimension of each kernel. Numbers of rows. kernel_c = self.__conv_kernel_dimensions__[ 0] # Second dimension of each kernel. Numbers of columns. for (i, num) in enumerate(self.__conv_layers_num_kernels__): if i == 0: prev_dim = 1 prev_pool = self.__conv_input_batch__ else: prev_dim = self.__conv_layers_num_kernels__[i - 1] prev_pool = self.__conv_pools__[i - 1] self.__conv_weights__.append( tf.Variable( tf.truncated_normal([kernel_r, kernel_c, prev_dim, num], stddev=0.1))) # self.__conv_biases__.append(tf.Variable(tf.constant(0.1, shape=[num]))) self.__conv_biases__.append( tf.Variable(tf.truncated_normal(shape=[num], stddev=0.1))) self.__conv_outputs__.append( tf.nn.conv2d(prev_pool, self.__conv_weights__[i], strides=[1, 1, 1, 1], padding=ConvNetwork.CONST_PADDING) + self.__conv_biases__[i]) self.__conv_activations__.append( tf.nn.relu(self.__conv_outputs__[i])) self.__conv_pools__.append( tf.nn.max_pool(self.__conv_activations__[i], ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding=ConvNetwork.CONST_PADDING))
def body(self, features): """TextCNN main model_fn. Args: features: Map of features to the model. Should contain the following: "inputs": Text inputs. [batch_size, input_length, 1, hidden_dim]. "targets": Target encoder outputs. [batch_size, 1, 1, hidden_dim] Returns: Final encoder representation. [batch_size, 1, 1, hidden_dim] """ hparams = self._hparams inputs = features["inputs"] xshape = common_layers.shape_list(inputs) vocab_size = xshape[3] inputs = tf.reshape(inputs, [xshape[0], xshape[1], xshape[3], xshape[2]]) pooled_outputs = [] for _, d_ff in enumerate(hparams.d_ffs): with tf.name_scope("conv-maxpool-%s" % d_ff): filter_shape = [d_ff, vocab_size, 1, hparams.num_filters] filter_var = tf.Variable( tf.truncated_normal(filter_shape, stddev=0.1), name="W") filter_bias = tf.Variable( tf.constant(0.1, shape=[hparams.num_filters]), name="b") conv = tf.nn.conv2d( inputs, filter_var, strides=[1, 1, 1, 1], padding="VALID", name="conv") conv_outputs = tf.nn.relu( tf.nn.bias_add(conv, filter_bias), name="relu") pooled = tf.math.reduce_max( conv_outputs, axis=1, keepdims=True, name="max") pooled_outputs.append(pooled) num_filters_total = hparams.num_filters * len(hparams.d_ffs) h_pool = tf.concat(pooled_outputs, 3) h_pool_flat = tf.reshape(h_pool, [-1, num_filters_total]) # Add dropout output = tf.nn.dropout(h_pool_flat, 1 - hparams.output_dropout) output = tf.reshape(output, [-1, 1, 1, num_filters_total]) return output
def conv2d(input): # Filter (weights and bias) # The shape of the filter weight is (height, width, input_depth, output_depth) # The shape of the filter bias is (output_depth,) # TODO: Define the filter weights `F_W` and filter bias `F_b`. # NOTE: Remember to wrap them in `tf.Variable`, they are trainable parameters after all. F_W = tf.Variable(tf.truncated_normal((2, 2, 1, 3))) F_b = tf.Variable(tf.zeros(3)) # TODO: Set the stride for each dimension (batch_size, height, width, depth) strides = [1, 2, 2, 1] # TODO: set the padding, either 'VALID' or 'SAME'. padding = 'VALID' # https://www.tensorflow.org/versions/r0.11/api_docs/python/nn.html#conv2d # `tf.nn.conv2d` does not include the bias computation so we have to add it ourselves after. return tf.nn.conv2d(input, F_W, strides, padding) + F_b
def fc_layer(self,idx,inputs,hiddens,flat = False,linear = False): input_shape = inputs.get_shape().as_list() if flat: dim = input_shape[1]*input_shape[2]*input_shape[3] inputs_transposed = tf.transpose(inputs,(0,3,1,2)) inputs_processed = tf.reshape(inputs_transposed, [-1,dim]) else: dim = input_shape[1] inputs_processed = inputs weight = tf.Variable(tf.truncated_normal([dim,hiddens], stddev=0.1)) biases = tf.Variable(tf.constant(0.1, shape=[hiddens])) print ('Layer %d : Type = Full, Hidden = %d, Input dimension = %d, Flat = %d, Activation = %d' % (idx,hiddens,int(dim),int(flat),1-int(linear)) ) if linear : return tf.add(tf.matmul(inputs_processed,weight),biases,name=str(idx)+'_fc') ip = tf.add(tf.matmul(inputs_processed,weight),biases) return tf.maximum(self.alpha*ip,ip,name=str(idx)+'_fc')
def model(batch_x): #weights of neural network weights = { 'w1': tf.Variable(tf.truncated_normal([n_input, n_hidden1], stddev=0.1), name="w1"), 'w2': tf.Variable(tf.truncated_normal([n_hidden1, n_hidden2], stddev=0.1), name="w2"), 'w3': tf.Variable(tf.truncated_normal([n_hidden2, n_hidden3], stddev=0.1), name="w3"), 'out': tf.Variable(tf.truncated_normal([n_hidden3, n_class], stddev=0.1), name="wout"), } #biases of neural network biases = { 'b1': tf.Variable(tf.zeros([n_hidden1]), name="b1"), 'b2': tf.Variable(tf.zeros([n_hidden2]), name="b2"), 'b3': tf.Variable(tf.zeros([n_hidden3]), name="b3"), 'out': tf.Variable(tf.zeros([n_class]), name="bout") } #neural netork operations #layer1 layer_1 = tf.add(tf.matmul(batch_x, weights['w1']), biases['b1']) layer_1 = tf.maximum(0.0, layer_1) #layer2 layer_2 = tf.add(tf.matmul(layer_1, weights['w2']), biases['b2']) layer_2 = tf.maximum(0.0, layer_2) #layer3 layer_3 = tf.add(tf.matmul(layer_2, weights['w3']), biases['b3']) layer_3 = tf.maximum(0.0, layer_3) #output output_layer = tf.matmul(layer_3, weights['out']) + biases['out'] return output_layer
def phi_R(B): ''' The phi_R function that predict the effect of each interaction by applying f_R to each column of B ''' h_size = 50 B_trans = tf.transpose(B, [0, 2, 1]) B_trans = tf.reshape(B_trans, [-1, (Ds + Dr)]) w1 = tf.Variable(tf.random.truncated_normal([(Ds + Dr), h_size], stddev=0.1), name="r_w1", dtype=tf.float32) b1 = tf.Variable(tf.zeros([h_size]), name="r_b1", dtype=tf.float32) h1 = tf.nn.relu(tf.matmul(B_trans, w1) + b1) w2 = tf.Variable(tf.random.truncated_normal([h_size, h_size], stddev=0.1), name="r_w2", dtype=tf.float32) b2 = tf.Variable(tf.zeros([h_size]), name="r_b2", dtype=tf.float32) h2 = tf.nn.relu(tf.matmul(h1, w2) + b2) w3 = tf.Variable(tf.random.truncated_normal([h_size, h_size], stddev=0.1), name="r_w3", dtype=tf.float32) b3 = tf.Variable(tf.zeros([h_size]), name="r_b3", dtype=tf.float32) h3 = tf.nn.relu(tf.matmul(h2, w3) + b3) w4 = tf.Variable(tf.truncated_normal([h_size, h_size], stddev=0.1), name="r_w4", dtype=tf.float32) b4 = tf.Variable(tf.zeros([h_size]), name="r_b4", dtype=tf.float32) h4 = tf.nn.relu(tf.matmul(h3, w4) + b4) w5 = tf.Variable(tf.truncated_normal([h_size, De], stddev=0.1), name="r_w5", dtype=tf.float32) b5 = tf.Variable(tf.zeros([De]), name="r_b5", dtype=tf.float32) h5 = tf.matmul(h4, w5) + b5 h5_trans = tf.reshape(h5, [-1, Nr, De]) h5_trans = tf.transpose(h5_trans, [0, 2, 1]) return h5_trans
def linear_network(x_in: int, in_dim: int, out_dim: int, mu_in=mu, sigma_in=sigma): """ Create a linear network layer with the input parameters provided. """ W = tf.Variable( tf.truncated_normal(shape=(in_dim, out_dim), mean=mu_in, stddev=sigma_in)) B = tf.Variable(tf.zeros(shape=(1, out_dim))) y_out = tf.matmul(x_in, W) + B return y_out
def build_model(num_features, num_classes): # Number of features n = num_features #Number of classes. K = num_classes # There are n columns in the feature matrix X = tf.placeholder(tf.float32, [None, n]) # Label is mxK matrix. Y = tf.placeholder(tf.float32, [None, K]) # Weights. nxK matrix W = tf.Variable(tf.truncated_normal([n, K], stddev=0.001)) # Bias. One for each class. b = tf.Variable(tf.truncated_normal([K], stddev=0.001)) # Hypothesis uses softmax. #It is a mxK matrix logits = tf.matmul(X, W) + b Y_hat = tf.nn.softmax(logits) # Cost function for softmax hypotheses function cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=Y, logits=logits)) # Gradient Descent Optimizer optimizer = tf.train.GradientDescentOptimizer(0.001).minimize(cost) #Index of the highest predicted value is taken as #the final decision. correct_prediction = tf.equal(tf.argmax(Y_hat, 1), tf.argmax(Y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) return optimizer, X, Y, Y_hat, cost, accuracy
def inception_v3(nlabels, images): batch_norm_params = { "is_training": False, "trainable": True, "decay": 0.9997, "epsilon": 0.001, "variables_collections": { "beta": None, "gamma": None, "moving_mean": ["moving_vars"], "moving_variance": ["moving_vars"], } } weight_decay = 0.00004 stddev = 0.1 weights_regularizer = tf_slim.l2_regularizer(weight_decay) args_for_scope = ( dict(list_ops_or_scope=[tf_slim.layers.conv2d, tf_slim.layers.fully_connected], weights_regularizer=weights_regularizer, trainable=True), dict(list_ops_or_scope=[tf_slim.layers.conv2d], weights_initializer=tf1.truncated_normal_initializer(stddev=stddev), activation_fn=tf1.nn.relu, normalizer_fn=tf_slim.layers.batch_norm, normalizer_params=batch_norm_params), ) with tf1.variable_scope("InceptionV3", "InceptionV3", [images]) as scope, \ tf_slim.arg_scope(**args_for_scope[0]), \ tf_slim.arg_scope(**args_for_scope[1]): net, end_points = inception_v3_base(images, scope=scope) with tf1.variable_scope("logits"): shape = net.get_shape() net = tf_slim.layers.avg_pool2d(net, shape[1:3], padding="VALID", scope="pool") net = tf1.nn.dropout(net, 1, name='droplast') net = tf_slim.layers.flatten(net, scope="flatten") with tf1.variable_scope('output') as scope: weights = tf1.Variable( tf1.truncated_normal([2048, nlabels], mean=0.0, stddev=0.01), name='weights') biases = tf1.Variable( tf1.constant(0.0, shape=[nlabels], dtype=tf1.float32), name='biases') output = tf1.add(tf1.matmul(net, weights), biases, name=scope.name) tensor_name = re.sub('tower_[0-9]*/', '', output.op.name) tf1.summary.histogram(tensor_name + '/activations', output) tf1.summary.scalar(tensor_name + '/sparsity', tf1.nn.zero_fraction(output)) return output
def truncated_normal(size, std): # We use TF to implement initialization function for neural network weight because: # 1. Pytorch doesn't support truncated normal # 2. This specific type of initialization is important for rapid progress early in training in cartpole # Do not allow tf to use gpu memory unnecessarily cfg = tf.ConfigProto() cfg.gpu_options.allow_growth = True sess = tf.Session(config=cfg) val = sess.run(tf.truncated_normal(shape=size, stddev=std)) # Close the session and free resources sess.close() return torch.tensor(val, dtype=torch.float32)
def model(images,add_dropout = True): mu = 0 sigma = 0.1 dropout = 0.5 conv1_W = tf.Variable(tf.truncated_normal(shape = (5,5,1,12),mean = mu,stddev=sigma)) conv1_b = tf.Variable(tf.zeros(12)) conv1 = tf.nn.conv2d(images,conv1_W,strides=[1,1,1,1],padding='VALID') + conv1_b conv1 = tf.nn.relu(conv1) conv1 = tf.nn.max_pool(conv1,ksize=[1,2,2,1],strides=[1,2,2,1],padding='VALID',name = 'conv1') #14x14x12 conv2_W = tf.Variable(tf.truncated_normal(shape = (5,5,12,24),mean = mu,stddev=sigma)) conv2_b = tf.Variable(tf.zeros(24)) conv2 = tf.nn.conv2d(conv1,conv2_W,strides = [1,1,1,1],padding='VALID',name = 'conv2') + conv2_b conv2 = tf.nn.relu(conv2) conv2 = tf.nn.max_pool(conv2,ksize=[1,2,2,1],strides=[1,2,2,1],padding='VALID') #5,5,24 #fc0 = flatten(conv2) #conv=array(conv2) #fc0=conv2.flatten fc0=tf.layers.Flatten()(conv2) #fc0=tf.Variable(fc0) #600 fc1_W = tf.Variable(tf.truncated_normal(shape = (600,400),mean = mu,stddev = sigma)) fc1_b = tf.Variable(tf.zeros(400)) fc1 = tf.matmul(fc0,fc1_W) + fc1_b fc1 = tf.nn.relu(fc1) if add_dropout: fc1 = tf.nn.dropout(fc1,dropout) fc2_W = tf.Variable(tf.truncated_normal(shape=(400, 120), mean = mu, stddev = sigma)) fc2_b = tf.Variable(tf.zeros(120)) fc2 = tf.matmul(fc1, fc2_W) + fc2_b fc2 = tf.nn.relu(fc2) if add_dropout: fc2 = tf.nn.dropout(fc2,dropout) fc3_W = tf.Variable(tf.truncated_normal(shape = (120,84),mean = mu,stddev=sigma)) fc3_b = tf.Variable(tf.zeros(84)) fc3 = tf.matmul(fc2,fc3_W) +fc3_b fc3 = tf.nn.relu(fc3) if add_dropout: fc3 = tf.nn.dropout(fc3,dropout) fc4_w = tf.Variable(tf.truncated_normal(shape = (84,43),mean=mu,stddev=sigma)) fc4_b = tf.Variable(tf.zeros(43)) logits = tf.matmul(fc3,fc4_w) + fc4_b return logits
def convolutional_network(x_in: int, in_h_w: int, in_depth: int, filter_h_w: int, out_depth: int): """ Create a convolutional network layer with the input parameters provided. """ out_h_w = (in_h_w - filter_h_w) + 1 # no padding, stride = 1 W = tf.Variable( tf.truncated_normal(shape=(filter_h_w, filter_h_w, in_depth, out_depth), mean=mu, stddev=sigma)) B = tf.Variable(tf.zeros(shape=(1, out_h_w, out_h_w, out_depth))) strides = [1, 1, 1, 1] padding = "VALID" y_out = tf.nn.conv2d(x_in, W, strides=strides, padding=padding) + B return y_out
def sample_encoded_context(self, embeddings): '''Helper function for init_opt''' c_mean_logsigma = self.model.generate_condition(embeddings) mean = c_mean_logsigma[0] if cfg.TRAIN.COND_AUGMENTATION: # epsilon = tf.random_normal(tf.shape(mean)) epsilon = tf.truncated_normal(tf.shape(mean)) stddev = tf.exp(c_mean_logsigma[1]) c = mean + stddev * epsilon kl_loss = KL_loss(c_mean_logsigma[0], c_mean_logsigma[1]) else: c = mean kl_loss = 0 return c, cfg.TRAIN.COEFF.KL * kl_loss
def get_synthetic_inputs(self, input_name, nclass): # Synthetic input should be within [0, 255]. image_shape, label_shape = self.get_input_shapes('train') inputs = tf.truncated_normal(image_shape, dtype=self.data_type, mean=127, stddev=60, name=self.model_name + '_synthetic_inputs') inputs = contrib_framework.local_variable(inputs, name=input_name) labels = tf.random_uniform(label_shape, minval=0, maxval=nclass - 1, dtype=tf.int32, name=self.model_name + '_synthetic_labels') return (inputs, labels)
def fcn_layer(inputs, input_dim, output_dim, activation=None): """ inputs:输入数据 input_dim:输入神经元数量 output_dim:输出神经元数量 activation:激活函数 """ W = tf.Variable(tf.truncated_normal([input_dim, output_dim], stddev=0.1)) b = tf.Variable(tf.zeros([output_dim])) XWb = tf.matmul(inputs, W) + b #矩阵相乘 outputs = XWb if (activation is None) else activation(XWb) return outputs
def train_overview_RNN(df): df_x, df_y = arrange(df) vocab, max_sentence_len = get_vocab( df_x) # get all unique words in the whole overviews num_options = len(vocab) num_features = max_sentence_len batch_size = 15 x_train, x_test, y_train, y_test = train_test_split(df_x, df_y, test_size=0.2, random_state=1) num_input_examples = x_train.shape[ 0] # number of examples = number of rows in x_train cellsize = 30 x = tf.placeholder(tf.float32, [None, num_features, num_options]) y = tf.placeholder(tf.float32, [None, 1]) lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(cellsize, forget_bias=0.0) output, _ = tf.nn.dynamic_rnn(lstm_cell, x, dtype=tf.float32) output = tf.transpose(output, [1, 0, 2]) last = output[-1] learn.models.linear_regression(last, y) W = tf.Variable(tf.truncated_normal([cellsize, num_options], stddev=0.1)) b = tf.Variable(tf.constant(0.1, shape=[num_options])) z = tf.matmul(last, W) + b res = tf.nn.softmax(z) cross_entropy = tf.reduce_mean( -tf.reduce_sum(y * tf.log(res), reduction_indices=[1])) train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy) sess = tf.InteractiveSession() sess.run(tf.global_variables_initializer()) correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(res, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) num_of_epochs = 100 for ephoch in range(num_of_epochs): acc = 0 curr_batch = 0 while True: batch_xs, batch_ys = next_batch(x_train, y_train, num_input_examples, batch_size, num_options, num_features, vocab) if batch_xs is None: break else: sess.run(train_step, feed_dict={x: batch_xs, y: batch_ys}) acc += accuracy.eval(feed_dict={x: batch_xs, y: batch_ys}) print("step %d, training accuracy %g" % (ephoch, acc / curr_batch))
def __init__(self, input_tensor, filter_shape: List[int], strides=None, padding="VALID"): if strides is None: strides = [1, 1, 1, 1] W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="weight") b = tf.Variable(tf.constant(0.1, shape=[filter_shape[-1]]), name="bias") conv = tf.nn.conv2d(input_tensor, W, strides=strides, padding=padding, name='conv') self.output = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
def testFlopRegularizerDontConvertToVariable(self): tf.reset_default_graph() tf.set_random_seed(1234) x = tf.constant(1.0, shape=[2, 6], name='x', dtype=tf.float32) w = tf.Variable(tf.truncated_normal([6, 4], stddev=1.0), use_resource=True) net = tf.matmul(x, w) # Create FLOPs network regularizer. threshold = 0.9 flop_reg = flop_regularizer.GroupLassoFlopsRegularizer([net.op], threshold, 0) with self.cached_session(): tf.global_variables_initializer().run() flop_reg.get_regularization_term().eval()
def __init__(self, n_in, n_out, name, x=None, W=None, b=None, shared_weights=True): super().__init__(n_in, n_out, name, x, W, b, tf.sigmoid) self.b_prime = tf.Variable(tf.zeros(shape=[n_in]), name='biases_prime') self.variables.append(self.b_prime) if shared_weights: self.W_prime = tf.transpose(self.W) else: self.W_prime = tf.Variable( tf.truncated_normal(shape=[n_out, n_in], stddev=1.0 / math.sqrt(float(n_in))), name='weights_prime' ) self.variables.append(self.W_prime) self.encode_logit = tf.matmul(self.y, self.W_prime) + self.b_prime self.encode = tf.sigmoid(self.encode_logit) self.encode_loss = tf.reduce_mean(tf.pow(self.x - self.encode, 2)) self.summaries.append(tf.summary.scalar(name+'/ae_rmse', self.encode_loss)) self.merged = tf.summary.merge(self.summaries)
def MLP_add_layer(train_data, input_size, output_size, activation_function=None): weights = tf.Variable( tf.truncated_normal([input_size, output_size], mean=0, stddev=0.1, dtype="float32")) biases = tf.Variable(tf.zeros([1, output_size], dtype="float32") + 0.1) input_mul_wb = tf.matmul(train_data, weights) + biases if activation_function is None: outputs = input_mul_wb else: outputs = activation_function(input_mul_wb) return outputs
def add_final_training_ops(class_count, final_tensor_name, bottleneck_tensor): with tf.name_scope('input'): bottleneck_input = tf.placeholder_with_default( bottleneck_tensor, shape=[None, BOTTLENECK_TENSOR_SIZE], name='BottleneckInputPlaceholder') ground_truth_input = tf.placeholder(tf.float32, [None, class_count], name='GroundTruthInput') # Organizing the following ops as `final_training_ops` so they're easier # to see in TensorBoard layer_name = 'final_training_ops' with tf.name_scope(layer_name): with tf.name_scope('weights'): initial_value = tf.truncated_normal( [BOTTLENECK_TENSOR_SIZE, class_count], stddev=0.001) layer_weights = tf.Variable(initial_value, name='final_weights') variable_summaries(layer_weights) with tf.name_scope('biases'): layer_biases = tf.Variable(tf.zeros([class_count]), name='final_biases') variable_summaries(layer_biases) with tf.name_scope('Wx_plus_b'): logits = tf.matmul(bottleneck_input, layer_weights) + layer_biases tf.summary.histogram('pre_activations', logits) final_tensor = tf.nn.softmax(logits, name=final_tensor_name) tf.summary.histogram('activations', final_tensor) with tf.name_scope('cross_entropy'): cross_entropy = tf.nn.softmax_cross_entropy_with_logits( labels=ground_truth_input, logits=logits) with tf.name_scope('total'): cross_entropy_mean = tf.reduce_mean(cross_entropy) tf.summary.scalar('cross_entropy', cross_entropy_mean) with tf.name_scope('train'): optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate) train_step = optimizer.minimize(cross_entropy_mean) return (train_step, cross_entropy_mean, bottleneck_input, ground_truth_input, final_tensor)
def __init__(self, flag_train, hps): self.flag_train = flag_train self.hps = hps self.activation = tf.nn.relu W, b = [], [] hps.n_hs_ext = [hps.n_in] + hps.n_hs + [hps.n_out] for i in range(len(hps.n_hs_ext) - 1): w_init = tf.truncated_normal( [hps.n_hs_ext[i], hps.n_hs_ext[i + 1]], mean=0, stddev=tf.sqrt(2.0 / hps.n_hs_ext[i]), seed=hps.seed) W.append(tf.Variable(w_init, name='weight_' + str(i))) b_init = tf.zeros([1, hps.n_hs_ext[i + 1]]) b.append(tf.Variable(b_init, name='bias_' + str(i))) self.W, self.b = W, b
def inference(images, train_phase): """Build the model up to where it may be used for inference. Args: images: Images placeholder. train_phase: Train phase placeholder Returns: logits: Output tensor with the computed logits. """ tn_init = lambda dev: lambda shape: tf.truncated_normal(shape, stddev=dev) tu_init = lambda bound: lambda shape: tf.random_uniform( shape, minval=-bound, maxval=bound) dropout_rate = lambda p: (opts['use_dropout'] * (p - 1.0)) * tf.to_float(train_phase) + 1.0 layers = [] layers.append(images) cnt = len(opts['hidden_units']) for i in range(cnt): n_out = opts['hidden_units'][i] layers.append( tensornet.layers.linear(layers[-1], n_out, scope='linear_' + str(len(layers)), biases_initializer=None)) layers.append( tensornet.layers.batch_normalization(layers[-1], train_phase, scope='BN_' + str(len(layers)), ema_decay=0.8)) layers.append(tf.nn.relu(layers[-1], name='relu_' + str(len(layers)))) layers.append( tf.nn.dropout(layers[-1], dropout_rate(0.77), name='dropout_' + str(len(layers)))) layers.append( tensornet.layers.linear(layers[-1], NUM_CLASSES, scope='linear_' + str(len(layers)))) return layers[-1]
def train(params=None): num_classes = 10 input_size = 784 hidden_units_size = 30 batch_size = 100 training_iterations = 10000 # 加载数据 mnist = input_data.read_data_sets('/storage/emulated/0/tensor-data/', one_hot=True) x_train = mnist.train.images y_train = mnist.train.labels x_test = mnist.test.images y_test = mnist.test.labels # 定义变量 x = tf.placeholder(tf.float32, shape=[None, input_size]) y_ = tf.placeholder(tf.float32, shape=[None, num_classes]) w = tf.Variable(tf.truncated_normal([input_size, num_classes]), name='w1') b = tf.Variable(tf.constant(0.01, shape=[num_classes]), name='b1') y = tf.nn.softmax(tf.matmul(x, w) + b) # 定义损失 loss = -tf.reduce_mean(y_ * tf.log(y)) optimizer = tf.train.GradientDescentOptimizer(0.2) train = optimizer.minimize(loss) # 初始化变量 sess = tf.Session() sess.run(tf.global_variables_initializer()) # 验证正确率 accuracy_rate = tf.equal(tf.arg_max(y, 1), tf.arg_max(y_, 1)) accuracy = tf.reduce_mean(tf.cast(accuracy_rate, 'float32')) # 训练 for step in range(training_iterations): batch_x, batch_y = mnist.train.next_batch(batch_size) sess.run(train, feed_dict={x: batch_x, y_: batch_y}) if step % 1000 == 0: print(step, sess.run(accuracy, feed_dict={x: x_test, y_: y_test})) if not tf.gfile.Exists('/storage/emulated/0/tensor-model/'): tf.gfile.MakeDirs('/storage/emulated/0/tensor-model/') saver = tf.train.Saver() # 保存模型 实例化 saver.save(sess, '/storage/emulated/0/tensor-model/my_model.ckpt')