def lenet(network_input: NetworkInput) -> KerasModel: model = Sequential() input_shape = network_input.input_shape if len(network_input.input_shape) < 3: model.add( layers.Lambda(lambda x: tf.expand_dims(x, -1), input_shape=input_shape)) input_shape = (input_shape[0], input_shape[1], 1) if network_input.mean is not None and network_input.std is not None: model.add( layers.Lambda( lambda x: norm(x, network_input.mean, network_input.std), input_shape=input_shape)) model.add( layers.Conv2D(32, kernel_size=(3, 3), input_shape=input_shape, activation='relu')) model.add(layers.Conv2D(64, (3, 3), activation='relu')) model.add(layers.MaxPooling2D(pool_size=(2, 2))) model.add(layers.Dropout(0.2)) model.add(layers.Flatten()) model.add(layers.Dense(128, activation='relu')) model.add(layers.Dropout(0.2)) model.add( layers.Dense(network_input.number_of_classes, activation='softmax')) return model
def resnet(num_blocks, img_input=None, classes=10, training=None): """Instantiates the ResNet architecture. Arguments: num_blocks: integer, the number of conv/identity blocks in each block. The ResNet contains 3 blocks with each block containing one conv block followed by (layers_per_block - 1) number of idenity blocks. Each conv/idenity block has 2 convolutional layers. With the input convolutional layer and the pooling layer towards the end, this brings the total size of the network to (6*num_blocks + 2) classes: optional number of classes to classify images into training: Only used if training keras model with Estimator. In other scenarios it is handled automatically. Returns: A Keras model instance. """ if backend.image_data_format() == 'channels_first': x = layers.Lambda(lambda x: backend.permute_dimensions(x, (0, 3, 1, 2)), name='transpose')(img_input) bn_axis = 1 else: # channel_last x = img_input bn_axis = 3 x = layers.ZeroPadding2D(padding=(1, 1), name='conv1_pad')(x) x = layers.Conv2D(16, (3, 3), strides=(1, 1), padding='valid', use_bias=False, kernel_initializer='he_normal', kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name='conv1')(x) x = layers.BatchNormalization(axis=bn_axis, momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON, name='bn_conv1',)(x, training=training) x = layers.Activation('relu')(x) x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[16, 16], stage=2, conv_strides=(1, 1), training=training) x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[32, 32], stage=3, conv_strides=(2, 2), training=training) x = resnet_block(x, size=num_blocks, kernel_size=3, filters=[64, 64], stage=4, conv_strides=(2, 2), training=training) rm_axes = [1, 2] if backend.image_data_format() == 'channels_last' else [2, 3] x = layers.Lambda(lambda x: backend.mean(x, rm_axes), name='reduce_mean')(x) x = layers.Dense(classes, activation='softmax', kernel_initializer=initializers.RandomNormal(stddev=0.01), kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY), bias_regularizer=regularizers.l2(L2_WEIGHT_DECAY), name='fc10')(x) inputs = img_input # Create model. model = tf.keras.models.Model(inputs, x, name='resnet56') return model
def __init__(self, tensors: MDPTensors, scale=None, **kwargs): super().__init__(tensors, name='reward_prediction', scale=scale) if self.loss is None: state_rep = tf.keras.Input( shape=self.tensors.state_representation.shape[1:], name='state_representation_input') act_in = tf.keras.Input(shape=self.tensors.action.shape[1:], name='action_input') rewards = tf.keras.Input(shape=self.tensors.reward.shape[1:], name='rewards_input') if kwargs['discrete_actions']: act = layers.Lambda(lambda x: tf.cast(x, tf.int32))(act_in) act = layers.Lambda(lambda x: tf.one_hot( x, depth=kwargs['n_actions'], dtype=tf.float32))(act) else: act = act_in merged = layers.concatenate([state_rep, act]) merged, = self.optional_gradient_stop(merged) x = layers.Dense(32, activation='elu')(merged) pred = layers.Dense(1, activation=None)(x) mse = layers.Lambda(lambda x: mean_squared_error(x[0], x[1]))( (rewards, pred)) mse = layers.Lambda(lambda x: backend.mean(x))(mse) scaled_mse = layers.Lambda(lambda x: x * self.scale)(mse) self.model = Model(inputs=[state_rep, act_in, rewards], outputs=[scaled_mse]) self.loss = self.model([ self.tensors.state_representation, self.tensors.action, self.tensors.reward ])
def convnet_preprocessor(input_shapes, image_shape, output_size, name="convnet_preprocessor", make_picklable=True, *args, **kwargs): inputs = [layers.Input(shape=input_shape) for input_shape in input_shapes] concatenated_input = layers.Lambda(lambda x: tf.concat(x, axis=-1))(inputs) image_size = np.prod(image_shape) images_flat, input_raw = layers.Lambda( lambda x: [x[..., :image_size], x[..., image_size:]])( concatenated_input) images = layers.Reshape(image_shape)(images_flat) preprocessed_images = convnet( input_shape=image_shape, output_size=output_size - input_raw.shape[-1], *args, **kwargs, )(images) output = layers.Lambda(lambda x: tf.concat(x, axis=-1))( [preprocessed_images, input_raw]) preprocessor = PicklableKerasModel(inputs, output, name=name) return preprocessor
def SplitAttentionConv2D(x, filters, kernel_size, stride=1, padding=(0, 0), groups=1, use_bias=True, radix=2, name=None): bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 reduction_factor = 4 inter_filters = max(filters * radix // reduction_factor, 32) x = layers.ZeroPadding2D((padding, padding), name=name + '_splat_pad')(x) x = layers.Conv2D(filters * radix, kernel_size=kernel_size, strides=stride, groups=groups * radix, use_bias=use_bias, name=name + '_0_splat_conv')(x) x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_0_splat_bn')(x) x = layers.Activation('relu', name=name + '_0_splat_relu')(x) splits = layers.Lambda(lambda x: tf.split(x, radix, bn_axis), name=name + '_0_splat_split')(x) x = layers.Add(name=name + '_0_splat_add')(splits) x = layers.GlobalAveragePooling2D(name=name + '_0_splat_pool')(x) shape = (1, 1, filters) if bn_axis == 3 else (filters, 1, 1) x = layers.Reshape(shape, name=name + '_0_splat_reshape')(x) x = layers.Conv2D(inter_filters, kernel_size=1, groups=groups, name=name + '_1_splat_conv')(x) x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name=name + '_1_splat_bn')(x) x = layers.Activation('relu', name=name + '_1_splat_relu')(x) # Attention x = layers.Conv2D(filters * radix, kernel_size=1, groups=groups, name=name + '_2_splat_conv')(x) x = RSoftmax(x, filters * radix, radix, groups, name=name) x = layers.Lambda(lambda x: tf.split(x, radix, bn_axis), name=name + '_1_splat_split')(x) x = layers.Lambda( lambda x: [tf.stack(x[0], axis=bn_axis), tf.stack(x[1], axis=bn_axis)], name=name + '_splat_stack')([splits, x]) x = layers.Multiply(name=name + '_splat_mult')(x) x = layers.Lambda(lambda x: tf.unstack(x, axis=bn_axis), name=name + '_splat_unstack')(x) x = layers.Add(name=name + '_splat_add')(x) return x
def make_parallel(keras_model, gpu_list): """Creates a new wrapper model that consists of multiple replicas of the original model placed on different GPUs. Args: keras_model: the input model to replicate on multiple gpus gpu_list: the number of replicas to build Returns: Multi-gpu model """ # Slice inputs. Slice inputs on the CPU to avoid sending a copy # of the full inputs to all GPUs. Saves on bandwidth and memory. gpu_list = [int(i) for i in gpu_list] input_slices = {name: tf.split(x, len(gpu_list)) for name, x in zip(keras_model.input_names, keras_model.inputs)} output_names = keras_model.output_names outputs_all = [] for i in range(len(keras_model.outputs)): outputs_all.append([]) # Run the model call() on each GPU to place the ops there for i in gpu_list: with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % i): # Run a slice of inputs through this replica zipped_inputs = zip(keras_model.input_names, keras_model.inputs) inputs = [ KL.Lambda(lambda s: input_slices[name][gpu_list.index(i)], output_shape=lambda s: (None,) + s[1:])(tensor) for name, tensor in zipped_inputs] # Create the model replica and get the outputs outputs = keras_model(inputs) if not isinstance(outputs, list): outputs = [outputs] # Save the outputs for merging back together later for l, o in enumerate(outputs): outputs_all[l].append(o) # Merge outputs on CPU with tf.device('/cpu:0'): merged = [] for outputs, name in zip(outputs_all, output_names): # Concatenate or average outputs? # Outputs usually have a batch dimension and we concatenate # across it. If they don't, then the output is likely a loss # or a metric value that gets averaged across the batch. # Keras expects losses and metrics to be scalars. if K.int_shape(outputs[0]) == (): # Average m = KL.Lambda(lambda o: tf.add_n( o) / len(outputs), name=name)(outputs) else: # Concatenate m = KL.Concatenate(axis=0, name=name)(outputs) merged.append(m) return merged
def test_model(self, strategy_fn, use_operator=False, use_regularizer=False, policy_name='mixed_float16', experimental_run_tf_function=True): if not self._is_strategy_supported(strategy_fn, check_model_type=True): return regularizer = IdentityRegularizer() if use_regularizer else None with strategy_fn().scope(): # Pass loss_scale=None, as this test will fail if the DynamicLossScale # skips applying gradients for a step with policy.policy_scope( policy.Policy(policy_name, loss_scale=None)): layer_list = [] if testing_utils.get_model_type() == 'subclass': # Subclassed models do not have an Input layer, so the model does not # cast inputs to the Input layer's dtype. Therefore, we need to # manually insert a float16 cast. cast_f16_layer = layers.Lambda( lambda x: math_ops.cast(x, 'float16'), input_shape=(1, )) layer_list.append(cast_f16_layer) layer = AddLayer(assert_type=dtypes.float16, use_operator=use_operator, regularizer=regularizer, input_shape=(1, )) cast_f32_layer = layers.Lambda( lambda x: math_ops.cast(x, 'float32')) layer_list += [layer, cast_f32_layer] model = testing_utils.get_model_from_layers( layer_list, input_shape=(1, ), input_dtype=dtypes.float16) def loss_fn(y_true, y_pred): del y_true return math_ops.reduce_mean(y_pred) # Learning rate is small enough that if applied to a float16 variable, # the variable will not change. So this tests the learning rate not # applied to a float16 value, but instead the float32 variable. opt = gradient_descent.SGD(2**-14) model.compile(opt, loss=loss_fn, run_eagerly=testing_utils.should_run_eagerly(), experimental_run_tf_function=testing_utils. should_run_tf_function()) x = np.ones((2, 1)) y = np.ones((2, 1)) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)).batch(2) model.fit(dataset) # Variable starts at 1, and should have gradient of 2 ** -14 subtracted # from it. expected = 1 - 2**-14 if use_regularizer: # Regularizer adds another 2 ** -14 to the gradient. expected -= 2**-14 self.assertEqual(backend.eval(layer.v), expected)
def define_rnn_network(hidden_state_dim, timepoints, timepoint_step): rnn_input = layers.Input(shape=(40, ), name="g_input") x = layers.Dense(64, activation="relu")(rnn_input) x = layers.Dense(64, activation="relu")(x) # The last output predicts the initial state for the RNN. gru_state = layers.Dense(hidden_state_dim, activation="relu")(x) # The RNN state is passed through a separate regressor to obtain the (mean, # scale) for each of the 4 dimensions. gru = layers.GRU(hidden_state_dim) intermediate_regressor = layers.Dense(64, activation="relu") mean_and_scale_regressor = layers.Dense(8, activation="linear", name="mean_and_scale_regressor") # Ensure the regressed scale is positive; also reshape to Bx8 -> Bx1x8. kSigmaMin = 1e-3 # avoid poor conditioning absolute_scale_op = layers.Lambda(lambda x: K.concatenate( (x[:, :4], K.abs(x[:, 4:]) + kSigmaMin), axis=1)[:, tf.newaxis, :]) output_regressor = lambda x: \ absolute_scale_op(mean_and_scale_regressor(intermediate_regressor(x))) current_output = output_regressor(gru_state) # predict the first output t = timepoint_step next_timepoint_idx = 0 outputs = [] while t <= timepoints[-1] or np.isclose(t, timepoints[-1]): if np.isclose(t, timepoints[next_timepoint_idx]): outputs.append(current_output) t = timepoints[next_timepoint_idx] # avoid any accumulative drift next_timepoint_idx += 1 if next_timepoint_idx == len(timepoints): break #current_output_with_time = layers.Lambda( # lambda x: K.concatenate((x, K.zeros_like(x)[:,:,:1] + t)))( # current_output) gru_state = gru(current_output, initial_state=gru_state) current_output = output_regressor(gru_state) t += timepoint_step assert (len(outputs) == len(timepoints)) # Join the T [Bx1x8] outputs into a Bx8xT tensor, then split in half down # the first axis and reform into a Bx4xTx2 tensor. def rejoin_op(x): x = K.stack(x, axis=-1) return K.stack((x[:, 0, :4, :], x[:, 0, 4:, :]), axis=-1) output = layers.Lambda(rejoin_op, name="transforms")(outputs) M = models.Model(inputs=[rnn_input], outputs=[output], name='rnn_regressor') return M
def __init__(self, out_channels, norm_layer, norm_kwargs, conv_trainable=True, **kwargs): super(ASPPPooling, self).__init__() self.gap = tf.keras.Sequential([ klayers.GlobalAveragePooling2D(), klayers.Lambda(lambda x: tf.keras.backend.expand_dims(x, 1)), klayers.Lambda(lambda x: tf.keras.backend.expand_dims(x, 1)), klayers.Conv2D(out_channels, kernel_size=1, kernel_initializer='he_uniform', use_bias=False, trainable=conv_trainable), norm_layer(**({} if norm_kwargs is None else norm_kwargs)), klayers.ReLU() ])
def lrn(x, alpha=1e-4, beta=0.75, k=2, n=5, name=None): """ Local response normalization layer. Parameters: ---------- x : keras.backend tensor/variable/symbol Input tensor/variable/symbol. alpha : float, 1e-4 Variance scaling parameter alpha in the LRN expression. beta : float, 0.75 Power parameter beta in the LRN expression. k : float, 2 Parameter k in the LRN expression. n : int, 5 Normalization window width in elements. name : str, default None Layer name. Returns ------- keras.backend tensor/variable/symbol Resulted tensor/variable/symbol. """ if K.backend() == "mxnet": from keras.backend.mxnet_backend import keras_mxnet_symbol, KerasSymbol import mxnet as mx @keras_mxnet_symbol def gluon_lrn(x, alpha, beta, k, n): if isinstance(x, KerasSymbol): x = x.symbol if isinstance(alpha, KerasSymbol): alpha = alpha.symbol if isinstance(beta, KerasSymbol): beta = beta.symbol if isinstance(k, KerasSymbol): k = k.symbol if isinstance(n, KerasSymbol): n = n.symbol return KerasSymbol( mx.sym.LRN(data=x, alpha=alpha, beta=beta, knorm=k, nsize=n)) x = nn.Lambda( lambda z: gluon_lrn(x=z, alpha=alpha, beta=beta, k=k, n=n))(x) else: import tensorflow as tf x = nn.Lambda(lambda z: tf.nn.lrn( input=z, depth_radius=n, bias=k, alpha=alpha, beta=beta, name=name) )(x) return x
def get_convolutional_network(shape, use_periodic_pad=False): inputs = Input(shape=(shape, shape, 1)) x = inputs if use_periodic_pad: x = layers.Lambda(periodic_pad)(x) x = layers.Conv2D(PARAMS["conv_start_filters"], (3, 3), activation="relu")(x) x = layers.MaxPooling2D((2, 2))(x) for i in range(1, PARAMS["conv_depth"]): x_1 = layers.Conv2D(PARAMS["conv_start_filters"] + i * PARAMS["conv_increment"], (3, 3), activation='relu')(x) x = layers.add([ layers.Conv2D( PARAMS["conv_start_filters"] + i * PARAMS["conv_increment"], (3, 3))(x), x_1 ]) x = layers.MaxPooling2D((2, 2))(x) x = layers.Flatten()(x) x = layers.Dense(128, activation='relu')(x) x = layers.Dropout(0.25)(x) x = layers.Dense(1, activation='sigmoid')(x) model = models.Model(inputs=inputs, outputs=x) return model
def layer(input_tensor): inp_ch = int(backend.int_shape(input_tensor)[-1] // groups) # input grouped channels out_ch = int(filters // groups) # output grouped channels blocks = [] for c in range(groups): slice_arguments = { 'start': c * inp_ch, 'stop': (c + 1) * inp_ch, 'axis': slice_axis, } x = layers.Lambda(slice_tensor, arguments=slice_arguments)(input_tensor) x = layers.Conv2D(out_ch, kernel_size, strides=strides, kernel_initializer=kernel_initializer, use_bias=use_bias, activation=activation, padding=padding)(x) blocks.append(x) x = layers.Concatenate(axis=slice_axis)(blocks) return x
def call(self, inputs): x = self.tcn_net(inputs) x = layers.Lambda(lambda tt: tt[:, -1, :])(x) #x = self.dnn_1(x) x = self.dnn_2(x) x = self.output_layer(x) return x
def flatten(x, reshape=False): """ Flattens the input to two dimensional. Parameters: ---------- x : keras.backend tensor/variable/symbol Input tensor/variable/symbol. reshape : bool, default False Whether do reshape instead of flatten. Returns ------- keras.backend tensor/variable/symbol Resulted tensor/variable/symbol. """ if not is_channels_first(): def channels_last_flatten(z): z = K.permute_dimensions(z, pattern=(0, 3, 1, 2)) z = K.reshape(z, shape=(-1, np.prod(K.int_shape(z)[1:]))) updateshape(z) return z return nn.Lambda(channels_last_flatten)(x) else: if reshape: x = nn.Reshape((-1, ))(x) else: x = nn.Flatten()(x) return x
def __init__(self, image_shape: tuple, **kwargs): conv_layers = [ ConvLayerConfig(stride=4, filter_size=8, nr_filters=32, activation='relu', batch_norm=False), ConvLayerConfig(stride=2, filter_size=4, nr_filters=64, activation='relu', batch_norm=False), ConvLayerConfig(stride=1, filter_size=3, nr_filters=64, activation='relu', batch_norm=False), ] rgb = layers.Input(shape=image_shape, name='rgb_input', dtype=tf.uint8) t = layers.Lambda(lambda x: K.cast(x, dtype='float32') / 255.)(rgb) for cl in conv_layers: t = layers.Conv2D( filters=cl.nr_filters, kernel_size=(cl.filter_size, cl.filter_size), strides=(cl.stride, cl.stride), activation=cl.activation, )(t) encoded = layers.Reshape(target_shape=(np.prod(t.shape[1:]), ))(t) self.model = tf.keras.Model(inputs=[rgb], outputs=[encoded]) self.embedding_size = 3136
def create_network(inp, channels, name, patch_size=70): with tf.name_scope(name): inp_layer = kl.Input(tensor=inp) # Discriminator layer = kl.Lambda(lambda x: tf.random_crop(x, [-1, patch_size, patch_size, channels]))(inp_layer) layer = kl.Conv2D(64, 4, padding="same", strides=2)(layer) layer = InstanceNormalization()(layer) layer = kl.LeakyReLU(0.2)(layer) layer = kl.Conv2D(128, 4, padding="same", strides=2)(layer) layer = InstanceNormalization()(layer) layer = kl.LeakyReLU(0.2)(layer) layer = kl.Conv2D(256, 4, padding="same", strides=2)(layer) layer = InstanceNormalization()(layer) layer = kl.LeakyReLU(0.2)(layer) layer = kl.Conv2D(512, 4, padding="same", strides=2)(layer) layer = InstanceNormalization()(layer) layer = kl.LeakyReLU(0.2)(layer) # D_out = kl.Conv2D(1, 4, activation="sigmoid", padding="same")(layer) D_out = kl.Conv2D(1, 4, padding="same")(layer) model = k.Model(inputs=inp_layer, outputs=D_out) return model
def create_res_dem_bc(self, kernel_initializer = 'he_normal', img_flat_len = 1024, only_emb = False): attr_input = layers.Input(shape = (50,), name = 'attr') word_emb = layers.Input(shape = (self.wv_len,), name = 'wv') imag_classifier = layers.Input(shape = (img_flat_len,), name = 'img') label = layers.Input(shape = (1,), name = 'label') attr_dense = layers.Dense(self.wv_len, use_bias = True, kernel_initializer=kernel_initializer, kernel_regularizer = l2(1e-4), name = 'attr_dense')(attr_input) ini_dem_model = self.create_dem_bc(kernel_initializer = 'he_normal', img_flat_len = img_flat_len, only_emb = True) ini_dem_model.load_weights('./only_emb.h5') ini_dem_model_part = Model(inputs = ini_dem_model.inputs[2], outputs = ini_dem_model.outputs[0]) ini_dem_model_part.trainable = False ini_attr_word_emb_dense = ini_dem_model_part([word_emb]) if only_emb: attr_word_emb = word_emb else: attr_word_emb = layers.Concatenate(name = 'attr_word_emb')([word_emb, attr_dense]) attr_word_emb_dense = self.full_connect_layer(attr_word_emb, hidden_dim = [ int(img_flat_len * 2), int(img_flat_len * 1.5), int(img_flat_len * 1.25), int(img_flat_len) ], \ activation = 'relu', resnet = False, drop_out_ratio = 0.2) attr_word_emb_dense = layers.Lambda(lambda x: x[0] + x[1])([attr_word_emb_dense, ini_attr_word_emb_dense]) attr_x_img = layers.Lambda(lambda x: x[0] * x[1], name = 'attr_x_img')([attr_word_emb_dense, imag_classifier]) # attr_x_img = layers.Concatenate(name = 'attr_x_img')([attr_word_emb_dense, imag_classifier]) attr_img_input = layers.Input(shape = (img_flat_len,), name = 'attr_img_input') # attr_img_input = layers.Input(shape = (img_flat_len * 2,), name = 'attr_img_input') proba = self.full_connect_layer(attr_img_input, hidden_dim = [1], activation = 'sigmoid') attr_img_model = Model(inputs = attr_img_input, outputs = proba, name = 'attr_x_img_model') out = attr_img_model([attr_x_img]) bc_loss = K.mean(binary_crossentropy(label, out)) model = Model([imag_classifier, attr_input, word_emb, label], outputs = [attr_word_emb_dense, out]) model.add_loss(bc_loss) model.compile(optimizer=Adam(lr=1e-4), loss=None) return model
def __init__(self, layer_sizes: List[int], layer_activations: List[Any], state_shape: tuple, action_shape: tuple, layer_and_batch_norm: bool, l2_param_penalty: float = 0.00, **kwargs): number_of_actions = action_shape[0] super().__init__(layer_sizes, layer_activations, state_shape, action_shape, 0, layer_and_batch_norm, l2_param_penalty=l2_param_penalty) assert len(action_shape) == 1 state = tf.keras.Input(shape=state_shape, name='observation_input') shared = state for idx in range(len(layer_sizes)): shared = self.layer_with_layer_norm(shared, idx, 'shared') shared_nog = layers.Lambda( lambda x: tf.keras.backend.stop_gradient(x))(shared) q_head = layers.Dense(units=number_of_actions, use_bias=True, activation=None, name='q_prediction', kernel_regularizer=self.pn, bias_regularizer=self.pn)(shared) q_n_head = layers.Dense(units=number_of_actions, use_bias=True, activation=None, name='q_noise_prediction', kernel_regularizer=self.pn, bias_regularizer=self.pn)(shared) p_head = layers.Dense(units=number_of_actions, use_bias=True, activation=None, name='policy', kernel_regularizer=self.pn, bias_regularizer=self.pn)( shared_nog) # TODO DIVERGE pn_head = layers.Dense(units=number_of_actions, use_bias=True, activation=None, name='noise_policy')(shared_nog) self.model = tf.keras.Model( inputs=[state], outputs=[q_head, q_n_head, p_head, pn_head])
def __init__(self, F, scale_f, *args, **kwargs): super(_ResBlock, self).__init__(*args, **kwargs) self.conv1 = layers.Conv2D(F, (3, 3), padding="same", activation='relu') self.conv2 = layers.Conv2D(F, (3, 3), padding="same") self.scale = layers.Lambda(lambda x: scale_f * x, name="scale") self.add = layers.Add(name="add")
def build_model(fragment_length, nb_filters, nb_output_bins, dilation_depth, nb_stacks, use_skip_connections, learn_all_outputs, _log, desired_sample_rate, use_bias, res_l2, final_l2): def residual_block(x): original_x = x # TODO: initalization, regularization? # Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet. tanh_out = CausalAtrousConvolution1D(nb_filters, 2, dilation_rate=2 ** i, padding='valid', causal=True, use_bias=use_bias, name='dilated_conv_%d_tanh_s%d' % (2 ** i, s), activation='tanh', kernel_regularizer=l2(res_l2))(x) sigm_out = CausalAtrousConvolution1D(nb_filters, 2, dilation_rate=2 ** i, padding='valid', causal=True, use_bias=use_bias, name='dilated_conv_%d_sigm_s%d' % (2 ** i, s), activation='sigmoid', kernel_regularizer=l2(res_l2))(x) x = layers.Multiply(name='gated_activation_%d_s%d' % (i, s))([tanh_out, sigm_out]) res_x = layers.Convolution1D(nb_filters, 1, padding='same', use_bias=use_bias, kernel_regularizer=l2(res_l2))(x) skip_x = layers.Convolution1D(nb_filters, 1, padding='same', use_bias=use_bias, kernel_regularizer=l2(res_l2))(x) res_x = layers.add([original_x, res_x]) return res_x, skip_x input = Input(shape=(fragment_length, nb_output_bins), name='input_part') out = input skip_connections = [] out = CausalAtrousConvolution1D(nb_filters, 2, dilation_rate=1, padding='valid', causal=True, name='initial_causal_conv' )(out) for s in range(nb_stacks): for i in range(0, dilation_depth + 1): out, skip_out = residual_block(out) skip_connections.append(skip_out) if use_skip_connections: out = layers.add(skip_connections) out = layers.Activation('relu')(out) out = layers.Convolution1D(nb_output_bins, 1, padding='same', kernel_regularizer=l2(final_l2))(out) out = layers.Activation('relu')(out) out = layers.Convolution1D(nb_output_bins, 1, padding='same')(out) if not learn_all_outputs: raise DeprecationWarning('Learning on just all outputs is wasteful, now learning only inside receptive field.') out = layers.Lambda(lambda x: x[:, -1, :], output_shape=(out._keras_shape[-1],))( out) # Based on gif in deepmind blog: take last output? out = layers.Activation('softmax', name="output_softmax")(out) model = Model(input, out) receptive_field, receptive_field_ms = compute_receptive_field() _log.info('Receptive Field: %d (%dms)' % (receptive_field, int(receptive_field_ms))) return model
def trivial_model(num_classes): """Trivial model for ImageNet dataset.""" input_shape = (224, 224, 3) img_input = layers.Input(shape=input_shape) x = layers.Lambda(lambda x: backend.reshape(x, [-1, 224 * 224 * 3]), name='reshape')(img_input) x = layers.Dense(num_classes, activation='softmax', name='fc1000')(x) return models.Model(img_input, x, name='trivial')
def __init__(self, tensors: MDPTensors, scale=None, **kwargs): super().__init__(tensors, name='auto_encoding_prediction', scale=scale) if self.loss is None: state_rep_in = tf.keras.Input( shape=self.tensors.state_representation.shape[1:], name='state_representation_input') state_rep, = self.optional_gradient_stop(state_rep_in) ob_shape = self.tensors.observation.shape[1:] if len(ob_shape) == 3: decoder = RGBDecoder( image_shape=ob_shape, embedding_size=self.tensors.state_representation.shape[1]) observation = tf.keras.Input(shape=ob_shape, name='observation_input', dtype=tf.uint8) target = layers.Lambda(lambda x: backend.cast( x, dtype='float32') / 127.5 - 1.)(observation) else: assert len(ob_shape) == 1 decoder = Sequential() decoder.add(layers.Dense(32, activation='elu')) decoder.add(layers.Dense(ob_shape[0], activation='tanh')) observation = tf.keras.Input(shape=ob_shape, name='observation_input', dtype=tf.float32) target = observation decoded = decoder(state_rep) mse = layers.Lambda(lambda x: mean_squared_error(x[0], x[1]))( (target, decoded)) mse = layers.Lambda(lambda x: backend.mean(x))(mse) scaled_mse = layers.Lambda(lambda x: x * self.scale)(mse) self.model = Model(inputs=[observation, state_rep_in], outputs=[scaled_mse]) self.loss = self.model( [self.tensors.observation, self.tensors.state_representation])
def __init__(self, tensors: MDPTensors, scale: Optional[float], **kwargs): super().__init__(tensors, 'value_function', scale) if self.loss is None: initial_loss = tf.keras.Input( shape=self.tensors.value_function_loss.shape, name='unscaled_value_function_loss') loss, = self.optional_gradient_stop(initial_loss) scaled_loss = layers.Lambda(lambda x: x * self.scale)(loss) self.model = Model(inputs=[initial_loss], outputs=[scaled_loss]) self.loss = self.model(tensors.value_function_loss)
def __init__(self, tensors: MDPTensors, scale=None, **kwargs): super().__init__(tensors, name='inverse_dynamics_prediction', scale=scale) if self.loss is None: state_rep = tf.keras.Input( shape=self.tensors.state_representation.shape[1:], name='state_representation_input') act_in = tf.keras.Input(shape=self.tensors.action.shape[1:], name='action_input') next_state_rep = tf.keras.Input( shape=self.tensors.next_state_representation.shape[1:], name='next_state_representation') merged = layers.concatenate([state_rep, next_state_rep]) merged, = self.optional_gradient_stop(merged) x = layers.Dense(64, activation='elu')(merged) if kwargs['discrete_actions']: pred = layers.Dense(kwargs['n_actions'], activation=None)(x) act = layers.Lambda(lambda x: tf.cast(x, tf.int32))(act_in) loss = layers.Lambda( lambda x: tf.nn.sparse_softmax_cross_entropy_with_logits( labels=x[1], logits=x[0]))([pred, act]) else: pred = layers.Dense(self.tensors.action.shape[1], activation='tanh')(x) loss = layers.Lambda(lambda x: mean_squared_error(x[0], x[1]))( (act_in, pred)) loss = layers.Lambda(lambda x: backend.mean(x))(loss) scaled_loss = layers.Lambda(lambda x: x * self.scale)(loss) self.model = Model(inputs=[state_rep, act_in, next_state_rep], outputs=[scaled_loss]) self.loss = self.model([ self.tensors.state_representation, self.tensors.action, self.tensors.next_state_representation ])
def RSoftmax(x, filters, radix, groups, name): bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 c = filters // radix // groups shape = (groups, radix, c) if bn_axis == 3 else (groups, radix, c) x = layers.Reshape(shape, name=name + '_0_attn_reshape')(x) x = layers.Lambda(lambda x: tf.transpose(x, (0, 2, 1, 3)), name=name + '_attn_transpose')(x) x = layers.Softmax(axis=1, name=name + '_attn_softmax')(x) shape = (1, 1, filters) if bn_axis == 3 else (filters, 1, 1) x = layers.Reshape(shape, name=name + '_1_attn_reshape')(x) return x
def simple(network_input: NetworkInput) -> KerasModel: model = Sequential([ layers.Lambda(lambda x: norm(x, network_input.mean, network_input.std), input_shape=network_input.input_shape, output_shape=network_input.input_shape), layers.Flatten(), layers.Dense(128, activation='relu'), layers.Dropout(0.2), layers.Dense(units=network_input.number_of_classes, activation='softmax') ]) return model
def trivial_model(num_classes): """Trivial model for ImageNet dataset.""" input_shape = (224, 224, 3) img_input = layers.Input(shape=input_shape) x = layers.Lambda(lambda x: backend.reshape(x, [-1, 224 * 224 * 3]), name="reshape")(img_input) x = layers.Dense(1, name="fc1")(x) x = layers.Dense(num_classes, name="fc1000")(x) x = layers.Activation("softmax", dtype="float32")(x) return models.Model(img_input, x, name="trivial")
def discriminator_model(self): model = tf.keras.Sequential() # Convolution block 1 model.add(kl.Conv3D(filters=self.crop_size, input_shape=(self.num_frames, self.crop_size, self.crop_size, 3), kernel_size=4, strides=2, padding='same', kernel_initializer=self.conv_init)) model.add(kl.Lambda(lambda x: tf.contrib.layers.layer_norm(x))) model.add(kl.LeakyReLU(.2)) # Convolution block 2 model.add(kl.Conv3D(filters=self.crop_size * 2, kernel_size=4, strides=2, padding='same', kernel_initializer=self.conv_init)) model.add(kl.Lambda(lambda x: tf.contrib.layers.layer_norm(x))) model.add(kl.LeakyReLU(.2)) # Convolution block 3 model.add(kl.Conv3D(filters=self.crop_size * 4, kernel_size=4, strides=2, padding='same', kernel_initializer=self.conv_init)) model.add(kl.Lambda(lambda x: tf.contrib.layers.layer_norm(x))) model.add(kl.LeakyReLU(.2)) # Convolution block 4 model.add(kl.Conv3D(filters=self.crop_size * 8, kernel_size=4, strides=2, padding='same', kernel_initializer=self.conv_init)) model.add(kl.Lambda(lambda x: tf.contrib.layers.layer_norm(x))) model.add(kl.LeakyReLU(.2)) # Convolution block 5 model.add(kl.Conv3D(filters=1, kernel_size=4, strides=2, padding='same', kernel_initializer=self.conv_init)) model.add(kl.LeakyReLU(.2)) # Linear block model.add(kl.Flatten()) model.add(kl.Dense(1, kernel_initializer=tf.keras.initializers.random_normal(stddev=0.01))) return model
def __init__(self, scale, out_channel, kernel_size, activation=None, *args, **kwargs): super(SubpixelLayer, self).__init__(*args, **kwargs) self.conv = layers.Conv2D(out_channel * scale**2, kernel_size, padding='same', activation=activation) self.subpixel = layers.Lambda(lambda x: tf.nn.depth_to_space(x, scale))
def create_dem_bc_aug(self, kernel_initializer = 'he_normal', img_flat_len = 1024, only_emb = False): attr_input = layers.Input(shape = (self.attr_len,), name = 'attr') word_emb = layers.Input(shape = (self.wv_len,), name = 'wv') img_input = layers.Input(shape = (self.pixel, self.pixel, 3)) label = layers.Input(shape = (1,), name = 'label') # img_flat_model = Model(inputs = self.img_model[0].inputs, outputs = self.img_model[0].get_layer(name = 'avg_pool').output) imag_classifier = self.img_flat_model(img_input) if self.attr_emb_transform == 'flat': attr_emb = layers.Embedding(294, self.attr_emb_len)(attr_input) attr_dense = layers.Flatten()(attr_emb) #layers.GlobalAveragePooling1D()(attr_emb) elif self.attr_emb_transform == 'dense': attr_dense = layers.Dense(self.attr_emb_len, use_bias = True, kernel_initializer=kernel_initializer, kernel_regularizer = l2(1e-4), name = 'attr_dense')(attr_input) if only_emb: attr_word_emb = word_emb else: attr_word_emb = layers.Concatenate(name = 'attr_word_emb')([word_emb, attr_dense]) attr_word_emb_dense = self.full_connect_layer(attr_word_emb, hidden_dim = [ # int(img_flat_len * 4), int(img_flat_len * 2), int(img_flat_len * 1.5), int(img_flat_len * 1.25), # int(img_flat_len * 1.125), int(img_flat_len) ], \ activation = 'relu', resnet = False, drop_out_ratio = 0.2) # attr_word_emb_dense = self.full_connect_layer(attr_word_emb_dense, hidden_dim = [img_flat_len], # activation = 'relu') attr_x_img = layers.Lambda(lambda x: x[0] * x[1], name = 'attr_x_img')([attr_word_emb_dense, imag_classifier]) # attr_x_img = layers.Concatenate(name = 'attr_x_img')([attr_word_emb_dense, imag_classifier]) attr_img_input = layers.Input(shape = (img_flat_len,), name = 'attr_img_input') # attr_img_input = layers.Input(shape = (img_flat_len * 2,), name = 'attr_img_input') proba = self.full_connect_layer(attr_img_input, hidden_dim = [1], activation = 'sigmoid') attr_img_model = Model(inputs = attr_img_input, outputs = proba, name = 'attr_x_img_model') out = attr_img_model([attr_x_img]) # dem_bc_model = self.create_dem_bc(kernel_initializer = 'he_normal', # img_flat_len = img_flat_len, # only_emb = only_emb) # attr_word_emb_dense, out = dem_bc_model([imag_classifier, attr_input, word_emb, label]) bc_loss = K.mean(binary_crossentropy(label, out)) model = Model([img_input, attr_input, word_emb, label], outputs = [attr_word_emb_dense, out, imag_classifier]) model.add_loss(bc_loss) model.compile(optimizer=Adam(lr=1e-4), loss=None) return model