def build_network(x, n_layers, hidden_layer_nodes, keep_prob, training_phase): assert n_layers == len( hidden_layer_nodes ), 'Specified layer nodes and number of layers do not correspond.' layers = [x] with tf.variable_scope('BN_layers') as scope: hidden_1 = BN_layer_ops( x, shape=[config.n_features, hidden_layer_nodes[0]], name='BNhidden0', keep_prob=keep_prob, phase=training_phase) layers.append(hidden_1) for n in range(0, n_layers - 1): hidden_n = BN_layer_ops( layers[-1], shape=[hidden_layer_nodes[n], hidden_layer_nodes[n + 1]], name='BNhidden{}'.format(n + 1), keep_prob=keep_prob, phase=training_phase) layers.append(hidden_n) readout = readout_ops(layers[-1], shape=[hidden_layer_nodes[-1], config.n_classes], name='readout') return readout
def began_autoencoder(x, l2_penalty=0.0, n_hidden=128, n_output=64, out_fn=None, trainable=True): l = x layers = [] act_fn = tf.nn.elu n_repeat = int(np.log2(int(l.get_shape()[0]) / 8)) + 1 reg = tf.contrib.layers.l2_regularizer( l2_penalty) if l2_penalty > 0.0 else None init = get_initializer(act_fn) l = tf.layers.conv2d(l, n_hidden, 3, 1, activation=act_fn, padding='same', trainable=trainable, kernel_initializer=init) layers.append(l) for idx in range(n_repeat): n_channel = n_hidden * (idx + 1) l = tf.layers.conv2d(l, n_channel, 3, 1, activation=act_fn, padding='same', trainable=trainable, kernel_regularizer=reg, kernel_initializer=init) layers.append(l) l = tf.layers.conv2d(l, n_channel, 3, 1, activation=act_fn, padding='same', trainable=trainable, kernel_regularizer=reg, kernel_initializer=init) layers.append(l) if idx < n_repeat - 1: l = tf.layers.conv2d(l, n_channel, 3, 2, activation=act_fn, padding='same', trainable=trainable, kernel_regularizer=reg, kernel_initializer=init) layers.append(l) l = tf.reshape(l, [tf.shape(l)[0], np.prod(l.get_shape().as_list()[1:])]) l = tf.layers.dense(l, n_output, activation=out_fn) layers.append(l) return layers
def my_discriminator(l, l2_penalty, n_output, n_hidden=64, act_fn=lambda x: tf.maximum(0.2 * x, x), out_fn=None, use_batchnorm=True): # NOTE: this is built after the figure in the DCGAN paper, but the # actual published code used a different arch reg = tf.contrib.layers.l2_regularizer( l2_penalty) if l2_penalty > 0.0 else None layers = [] n_repeat = int(np.log2(int(l.get_shape()[0]) / 8)) for i in range(n_repeat): with tf.name_scope('disc.%02d' % i): l = tf.layers.conv2d(l, n_hidden, kernel_size=5, strides=2, padding='same', activation=None, kernel_regularizer=reg) if use_batchnorm: l = tf.layers.batch_normalization(l, training=True) l = act_fn(l) layers.append(l) n_hidden *= 2 l = tf.reshape(l, [tf.shape(l)[0], np.prod(l.get_shape().as_list()[1:])]) l = tf.layers.dense(l, n_output, activation=out_fn) layers.append(l) return layers
def _make_layer(self, block, planes, blocks, shortcut_type, stride=1, training=False): downsample = None stride_p = stride # especially for downsample branch. if self.cnt < self.depth_3d: if self.cnt == 0: stride_p = 1 else: stride_p = [2, 2, 1] if stride != 1 or self.inplanes != planes * block.expansion: downsample = Sequential([ Conv3D(planes * block.expansion, filter_size=1, strides=stride_p, name='conv_down'), BatchNormal(name='batch_down', training=training), ]) else: if stride != 1 or self.inplanes != planes * block.expansion: downsample = Sequential([ Conv2D(planes * block.expansion, filter_size=1, strides=2, name='conv_down'), BatchNormal(name='batch_down', training=training) ]) layers = [] layers.append( block(planes, stride, downsample, n_s=self.cnt, depth_3d=self.depth_3d, ST_struc=self.ST_struc, name='Bottleneck0', training=training)) self.cnt += 1 self.inplanes = planes * block.expansion for i in range(1, blocks): layers.append( block(planes, n_s=self.cnt, depth_3d=self.depth_3d, ST_struc=self.ST_struc, name='Bottleneck{}'.format(i), training=training)) self.cnt += 1 return Sequential(layers)
def shallow_wide_cnn(x, filter_widths, filter_channel): layers = [] x_dim = x.get_shape()[2].value x_width = x.get_shape()[1].value for idx, filter_width in enumerate(filter_widths): with tf.variable_scope('filter_{}'.format(idx)) as scope: W = tf.get_variable(name='conv_W', shape=[filter_width, x_dim, filter_channel], initializer=initializers.get("glorot_uniform")) conved = tf.nn.conv1d(value=x, filters=W, stride=1, padding='VALID') pooled = tf.reduce_max(conved, axis=1) layers.append(pooled) return tf.concat(layers, axis=1)
def began_generator(z, img_shape, l2_penalty, act_fn=tf.nn.elu, out_fn=None): reg = tf.contrib.layers.l2_regularizer( l2_penalty) if l2_penalty > 0.0 else None layers = [] n_repeat = int(np.log2(img_shape[0] / 4)) n_hidden = 128 l = tf.layers.dense(z, 8 * 8 * n_hidden, activation=None, kernel_regularizer=reg) layers.append(l) l = tf.reshape(l, [-1, 8, 8, n_hidden]) for idx in range(n_repeat): l = tf.layers.conv2d(l, n_hidden, 3, 1, activation=act_fn, padding='same', kernel_regularizer=reg, kernel_initializer=get_initializer(act_fn)) layers.append(l) l = tf.layers.conv2d(l, n_hidden, 3, 1, activation=act_fn, padding='same', kernel_regularizer=reg, kernel_initializer=get_initializer(act_fn)) layers.append(l) if idx < n_repeat - 1: l = scale(l, 2) layers.append(l) l = tf.layers.conv2d(l, img_shape[-1], 3, 1, activation=out_fn, padding='same', kernel_regularizer=reg) layers.append(l) return layers
def dcgan_generator(z, img_shape, l2_penalty, act_fn=tf.nn.relu, out_fn=tf.nn.tanh, use_batchnorm=True): reg = tf.contrib.layers.l2_regularizer( l2_penalty) if l2_penalty > 0.0 else None layers = [] n_filter = 512 img_res = 1 l = tf.reshape(z, [-1, 1, 1, int(np.prod(z.get_shape()[1:]))]) while img_res < img_shape[0] // 4: s = 2 if n_filter != 512 else 1 # first stride is smaller p = 'same' if n_filter != 512 else 'valid' # first stride is smaller l = tf.layers.conv2d_transpose( l, filters=n_filter, kernel_size=4, strides=s, padding=p, activation=None, kernel_regularizer=reg, data_format='channels_last', kernel_initializer=get_initializer(act_fn)) if use_batchnorm: l = tf.layers.batch_normalization(l, training=True) l = act_fn(l) layers.append(l) img_res *= 2 n_filter //= 2 l = tf.layers.conv2d_transpose(l, filters=img_shape[-1], kernel_size=4, strides=2, padding='same', activation=out_fn, kernel_regularizer=reg, data_format='channels_last') layers.append(l) #print('GENERATOR:\n' + '\n'.join([str(l) for l in layers])) return layers
def network_builder(x, n_layers, hidden_layer_nodes, keep_prob, training_phase): assert n_layers == len( hidden_layer_nodes ), 'Specified layer nodes and number of layers do not correspond.' layers = [x] if config.builder == 'bn': print('Building ReLU + Batch-norm architecture') builder = BN_layer_ops elif config.builder == 'selu': print('Building SELU architecture') builder = hidden_SELU_ops elif config.builder == 'selu-bn': print('Building SELU + Batch-norm architecture') builder = SELU_BN_layer_ops else: print('Default architecture: SELU') builder = hidden_SELU_ops with tf.variable_scope('hidden_layers') as scope: hidden_1 = builder(x, shape=[config.n_features, hidden_layer_nodes[0]], name='hidden0', keep_prob=keep_prob, phase=training_phase) layers.append(hidden_1) for n in range(0, n_layers - 1): hidden_n = builder( layers[-1], shape=[hidden_layer_nodes[n], hidden_layer_nodes[n + 1]], name='hidden{}'.format(n + 1), keep_prob=keep_prob, phase=training_phase) layers.append(hidden_n) readout = readout_ops(layers[-1], shape=[hidden_layer_nodes[-1], config.n_classes], name='readout', initializer=SELU_initializer) return readout
def _build(self, inputs): #max_word_length = inputs.get_shape()[1] max_word_length = tf.shape(inputs)[1]#xxx1 embed_size = inputs.get_shape()[-1] inputs = tf.expand_dims(inputs, 1) layers = [] self.conv_layers = [] for kernel_size, kernel_feature_size in zip(self._kernels, self._kernel_features): ## [batch_size x max_word_length x embed_size x kernel_feature_size] conv_fxn = snt.Conv2D(output_channels=kernel_feature_size, kernel_shape=[1, kernel_size],# ?? [kernel_size,1] ?? initializers=self._initializers, padding='VALID') conv = conv_fxn(inputs) self.conv_layers.append(conv_fxn) # ## [batch_size x 1 x 1 x kernel_feature_size] # reduced_length = max_word_length - kernel_size + 1 # pool = tf.nn.max_pool(tf.tanh(conv), # ksize= [1,1,reduced_length,1], # strides= [1,1,1,1], # padding= 'VALID') # https://stackoverflow.com/questions/43574076/tensorflow-maxpool-with-dynamic-ksize#xxx2 pool = tf.reduce_max(tf.tanh(conv), axis=2, keepdims=True ) layers.append(tf.squeeze(pool, [1, 2])) if len(self._kernels) > 1: output = tf.concat(layers, 1) else: output = layers[0] return output
def dc_bilstm(config, x, x_nums, rnn_cells): ''' Densely connected biLSTM ''' max_x_num = x.get_shape()[1] layers = [x] layer_num = len(rnn_cells) for i in range(layer_num): layers.append( bidirectional_rnn(rnn_cells[i], rnn_cells[i], tf.concat(layers, axis=2), x_nums, scope=None)[0]) pooled = tf.layers.average_pooling1d( inputs=layers[-1], pool_size=[1, max_x_num, 1], strides=1, ) return tf.reshape(pooled, [-1, pooled.get_shape()[-1]])
def dcgan_discriminator(l, l2_penalty, n_output, n_hidden=64, act_fn=lambda x: tf.maximum(0.2 * x, x), out_fn=None, use_batchnorm=True): reg = tf.contrib.layers.l2_regularizer( l2_penalty) if l2_penalty > 0.0 else None layers = [] n_repeat = int(np.log2(int(l.get_shape()[1]))) - 2 for i in range(n_repeat): l = tf.layers.conv2d(l, n_hidden, kernel_size=4, strides=2, padding='same', activation=None, kernel_regularizer=reg) if use_batchnorm: l = tf.layers.batch_normalization(l, training=True) l = act_fn(l) layers.append(l) n_hidden *= 2 l = tf.layers.conv2d(l, n_output, kernel_size=4, strides=1, padding='valid', activation=None, kernel_regularizer=reg) l = tf.squeeze(l) layers.append(l) #print('DISCRIMINATOR:\n' + '\n'.join([str(l) for l in layers])) return layers
def my_generator(z, img_shape, l2_penalty, act_fn=tf.nn.relu, out_fn=tf.nn.tanh, use_batchnorm=True): # NOTE: this is built after the figure in the DCGAN paper, but the # actual published code used a different arch reg = tf.contrib.layers.l2_regularizer( l2_penalty) if l2_penalty > 0.0 else None layers = [] n_filter = 1024 img_res = 4 l = tf.layers.dense(z, img_res * img_res * n_filter, activation=tf.nn.relu, name="layer0") l = tf.reshape(l, [-1, img_res, img_res, n_filter]) layers.append(l) i = 0 while img_res < img_shape[0] // 2: with tf.name_scope('gen.%02d' % i): n_filter //= 2 l = tf.layers.conv2d_transpose( l, filters=n_filter, kernel_size=5, strides=2, padding='same', activation=None, kernel_regularizer=reg, data_format='channels_last', kernel_initializer=get_initializer(act_fn)) if use_batchnorm: l = tf.layers.batch_normalization(l, training=True) l = act_fn(l) layers.append(l) img_res *= 2 i += 1 l = tf.layers.conv2d_transpose(l, filters=img_shape[-1], kernel_size=5, strides=2, padding='same', activation=out_fn, kernel_regularizer=reg, data_format='channels_last') layers.append(l) return layers
def vdcnn(x, filter_width=3, init_channel=64, num_layers=[2, 2, 2, 2], use_shortcut=False, k=8, is_training=True, scope=None): layers = [] x_dim = x.get_shape()[2] with tf.variable_scope("temp_conv"): filter_shape = [filter_width, x_dim, init_channel] W = tf.get_variable(name='temp_1', shape=filter_shape, initializer=initializers.get("glorot_uniform")) x = tf.nn.conv1d(x, W, stride=1, padding="SAME") layers.append(x) now_channel_size = init_channel for i, num_layer in enumerate(num_layers): for j in range(num_layer): with tf.variable_scope("%d_layer_%d_cnn" % (i, j)) as scope: shortcut = None if use_shortcut and i < len(num_layers) - 1: shortcut = layers[-1] conv_ = conv_block(layers[-1], shortcut, filter_width, now_channel_size, is_training, scope) layers.append(conv_) if i == len(num_layers) - 1: break with tf.variable_scope("%d_layer_pool" % (i)) as scope: shortcut = None if use_shortcut: shortcut = layers[-1] pool_ = pool_block(layers[-1], shortcut, filter_width, scope) layers.append(pool_) now_channel_size *= 2 k_pooled = tf.nn.top_k(tf.transpose(layers[-1], [0, 2, 1]), k=k, name='k_pool', sorted=False)[0] flatten = tf.reshape(k_pooled, (-1, now_channel_size * k)) return flatten
def preprocess_screen(screen): layers = [] assert screen.shape[0] == len(features.SCREEN_FEATURES) for i in range(len(features.SCREEN_FEATURES)): if i == _SCREEN_PLAYER_ID or i == _SCREEN_UNIT_TYPE: layers.append(screen[i:i+1] / features.SCREEN_FEATURES[i].scale) elif features.SCREEN_FEATURES[i].type == features.FeatureType.SCALAR: layers.append(screen[i:i+1] / features.SCREEN_FEATURES[i].scale) else: layer = np.zeros([features.SCREEN_FEATURES[i].scale, screen.shape[1], screen.shape[2]], dtype=np.float32) for j in range(features.SCREEN_FEATURES[i].scale): indy, indx = (screen[i] == j).nonzero() layer[j, indy, indx] = 1 layers.append(layer) return np.concatenate(layers, axis=0)
def preprocess_minimap(minimap): layers = [] assert minimap.shape[0] == len(features.MINIMAP_FEATURES) for i in range(len(features.MINIMAP_FEATURES)): if i == _MINIMAP_PLAYER_ID: layers.append(minimap[i:i+1] / features.MINIMAP_FEATURES[i].scale) elif features.MINIMAP_FEATURES[i].type == features.FeatureType.SCALAR: layers.append(minimap[i:i+1] / features.MINIMAP_FEATURES[i].scale) else: layer = np.zeros([features.MINIMAP_FEATURES[i].scale, minimap.shape[1], minimap.shape[2]], dtype=np.float32) for j in range(features.MINIMAP_FEATURES[i].scale): indy, indx = (minimap[i] == j).nonzero() layer[j, indy, indx] = 1 layers.append(layer) return np.concatenate(layers, axis=0)
def _build_ops(self, lm_graph): with tf.control_dependencies([lm_graph.update_state_op]): # get the LM embeddings token_embeddings = lm_graph.embedding layers = [ tf.concat([token_embeddings, token_embeddings], axis=2) ] n_lm_layers = len(lm_graph.lstm_outputs['forward']) for i in range(n_lm_layers): layers.append( tf.concat( [lm_graph.lstm_outputs['forward'][i], lm_graph.lstm_outputs['backward'][i]], axis=-1 ) ) # The layers include the BOS/EOS tokens. Remove them sequence_length_wo_bos_eos = lm_graph.sequence_lengths - 2 layers_without_bos_eos = [] for layer in layers: layer_wo_bos_eos = layer[:, 1:, :] layer_wo_bos_eos = tf.reverse_sequence( layer_wo_bos_eos, lm_graph.sequence_lengths - 1, seq_axis=1, batch_axis=0, ) layer_wo_bos_eos = layer_wo_bos_eos[:, 1:, :] layer_wo_bos_eos = tf.reverse_sequence( layer_wo_bos_eos, sequence_length_wo_bos_eos, seq_axis=1, batch_axis=0, ) layers_without_bos_eos.append(layer_wo_bos_eos) # concatenate the layers lm_embeddings = tf.concat( [tf.expand_dims(t, axis=1) for t in layers_without_bos_eos], axis=1 ) # get the mask op without bos/eos. # tf doesn't support reversing boolean tensors, so cast # to int then back mask_wo_bos_eos = tf.cast(lm_graph.mask[:, 1:], 'int32') mask_wo_bos_eos = tf.reverse_sequence( mask_wo_bos_eos, lm_graph.sequence_lengths - 1, seq_axis=1, batch_axis=0, ) mask_wo_bos_eos = mask_wo_bos_eos[:, 1:] mask_wo_bos_eos = tf.reverse_sequence( mask_wo_bos_eos, sequence_length_wo_bos_eos, seq_axis=1, batch_axis=0, ) mask_wo_bos_eos = tf.cast(mask_wo_bos_eos, 'bool') return { 'lm_embeddings': lm_embeddings, 'lengths': sequence_length_wo_bos_eos, 'token_embeddings': lm_graph.embedding, 'mask': mask_wo_bos_eos, }
def extract_features(self, input=None, reuse=True): if input is None: input = self.image x = input layers = [] with tf.variable_scope('features', reuse=reuse): with tf.variable_scope('layer0'): W = tf.get_variable("weights", shape=[3, 3, 3, 64]) b = tf.get_variable("bias", shape=[64]) x = tf.nn.conv2d(x, W, [1, 2, 2, 1], "VALID") x = tf.nn.bias_add(x, b) layers.append(x) with tf.variable_scope('layer1'): x = tf.nn.relu(x) layers.append(x) with tf.variable_scope('layer2'): x = tf.nn.max_pool(x, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID') layers.append(x) with tf.variable_scope('layer3'): x = fire_module(x, 64, 16, 64, 64) layers.append(x) with tf.variable_scope('layer4'): x = fire_module(x, 128, 16, 64, 64) layers.append(x) with tf.variable_scope('layer5'): x = tf.nn.max_pool(x, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID') layers.append(x) with tf.variable_scope('layer6'): x = fire_module(x, 128, 32, 128, 128) layers.append(x) with tf.variable_scope('layer7'): x = fire_module(x, 256, 32, 128, 128) layers.append(x) with tf.variable_scope('layer8'): x = tf.nn.max_pool(x, [1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID') layers.append(x) with tf.variable_scope('layer9'): x = fire_module(x, 256, 48, 192, 192) layers.append(x) with tf.variable_scope('layer10'): x = fire_module(x, 384, 48, 192, 192) layers.append(x) with tf.variable_scope('layer11'): x = fire_module(x, 384, 64, 256, 256) layers.append(x) with tf.variable_scope('layer12'): x = fire_module(x, 512, 64, 256, 256) layers.append(x) return layers