def build(self, hp, inputs=None): input_node = nest.flatten(inputs)[0] # TODO: support more pretrained embedding layers. # glove, fasttext, and word2vec pretraining = utils.add_to_hp(self.pretraining, hp) embedding_dim = utils.add_to_hp(self.embedding_dim, hp) if pretraining != "none": # TODO: load from pretrained weights layer = layers.Embedding( input_dim=self.max_features, output_dim=embedding_dim, input_length=input_node.shape[1], ) # trainable=False, # weights=[embedding_matrix]) else: layer = layers.Embedding( input_dim=self.max_features, output_dim=embedding_dim ) # input_length=input_node.shape[1], # trainable=True) output_node = layer(input_node) dropout = utils.add_to_hp(self.dropout, hp) if dropout > 0: output_node = layers.Dropout(dropout)(output_node) return output_node
def build(self, hp, inputs=None): inputs = nest.flatten(inputs) utils.validate_num_inputs(inputs, 1) input_node = inputs[0] output_node = input_node output_node = reduction.Flatten().build(hp, output_node) use_batchnorm = self.use_batchnorm if use_batchnorm is None: use_batchnorm = hp.Boolean("use_batchnorm", default=False) if self.dropout is not None: dropout = self.dropout else: dropout = hp.Choice("dropout", [0.0, 0.25, 0.5], default=0) for i in range(utils.add_to_hp(self.num_layers, hp)): units = utils.add_to_hp(self.num_units, hp, "units_{i}".format(i=i)) output_node = layers.Dense(units)(output_node) if use_batchnorm: output_node = layers.BatchNormalization()(output_node) output_node = layers.ReLU()(output_node) if dropout > 0: output_node = layers.Dropout(dropout)(output_node) return output_node
def build(self, hp, inputs=None): inputs = nest.flatten(inputs) utils.validate_num_inputs(inputs, 1) input_node = inputs[0] shape = input_node.shape.as_list() if len(shape) != 3: raise ValueError( "Expect the input tensor of RNNBlock to have dimensions of " "[batch_size, time_steps, vec_len], " "but got {shape}".format(shape=input_node.shape) ) feature_size = shape[-1] output_node = input_node bidirectional = utils.add_to_hp(self.bidirectional, hp) layer_type = utils.add_to_hp(self.layer_type, hp) num_layers = utils.add_to_hp(self.num_layers, hp) rnn_layers = {"gru": layers.GRU, "lstm": layers.LSTM} in_layer = rnn_layers[layer_type] for i in range(num_layers): return_sequences = True if i == num_layers - 1: return_sequences = self.return_sequences if bidirectional: output_node = layers.Bidirectional( in_layer(feature_size, return_sequences=return_sequences) )(output_node) else: output_node = in_layer( feature_size, return_sequences=return_sequences )(output_node) return output_node
def build(self, hp, inputs=None): """ # Arguments hp: HyperParameters. The hyperparameters for building the model. inputs: Tensor of Shape [batch_size, seq_len] # Returns Output Tensor of shape `[batch_size, seq_len, embedding_dim]`. """ inputs = nest.flatten(inputs) utils.validate_num_inputs(inputs, 1) pretraining = utils.add_to_hp(self.pretraining, hp) embedding_dim = utils.add_to_hp(self.embedding_dim, hp) num_heads = utils.add_to_hp(self.num_heads, hp) dense_dim = utils.add_to_hp(self.dense_dim, hp) dropout = utils.add_to_hp(self.dropout, hp) ffn = tf.keras.Sequential( [ layers.Dense(dense_dim, activation="relu"), layers.Dense(embedding_dim), ] ) layernorm1 = layers.LayerNormalization(epsilon=1e-6) layernorm2 = layers.LayerNormalization(epsilon=1e-6) dropout1 = layers.Dropout(dropout) dropout2 = layers.Dropout(dropout) # Token and Position Embeddings input_node = nest.flatten(inputs)[0] token_embedding = Embedding( max_features=self.max_features, pretraining=pretraining, embedding_dim=embedding_dim, dropout=dropout, ).build(hp, input_node) maxlen = input_node.shape[-1] batch_size = tf.shape(input_node)[0] positions = self.pos_array_funct(maxlen, batch_size) position_embedding = Embedding( max_features=maxlen, pretraining=pretraining, embedding_dim=embedding_dim, dropout=dropout, ).build(hp, positions) output_node = tf.keras.layers.Add()([token_embedding, position_embedding]) attn_output = MultiHeadSelfAttention(embedding_dim, num_heads).build( hp, output_node ) attn_output = dropout1(attn_output) add_inputs_1 = tf.keras.layers.Add()([output_node, attn_output]) out1 = layernorm1(add_inputs_1) ffn_output = ffn(out1) ffn_output = dropout2(ffn_output) add_inputs_2 = tf.keras.layers.Add()([out1, ffn_output]) return layernorm2(add_inputs_2)
def build(self, hp, inputs=None): input_node = nest.flatten(inputs)[0] output_node = input_node # Translate translation_factor = utils.add_to_hp(self.translation_factor, hp) if translation_factor not in [0, (0, 0)]: height_factor, width_factor = self._get_fraction_value( translation_factor ) output_node = layers.RandomTranslation(height_factor, width_factor)( output_node ) # Flip horizontal_flip = self.horizontal_flip if horizontal_flip is None: horizontal_flip = hp.Boolean("horizontal_flip", default=True) vertical_flip = self.vertical_flip if self.vertical_flip is None: vertical_flip = hp.Boolean("vertical_flip", default=True) if not horizontal_flip and not vertical_flip: flip_mode = "" elif horizontal_flip and vertical_flip: flip_mode = "horizontal_and_vertical" elif horizontal_flip and not vertical_flip: flip_mode = "horizontal" elif not horizontal_flip and vertical_flip: flip_mode = "vertical" if flip_mode != "": output_node = layers.RandomFlip(mode=flip_mode)(output_node) # Rotate rotation_factor = utils.add_to_hp(self.rotation_factor, hp) if rotation_factor != 0: output_node = layers.RandomRotation(rotation_factor)(output_node) # Zoom zoom_factor = utils.add_to_hp(self.zoom_factor, hp) if zoom_factor not in [0, (0, 0)]: height_factor, width_factor = self._get_fraction_value(zoom_factor) # TODO: Add back RandomZoom when it is ready. # output_node = layers.RandomZoom( # height_factor, width_factor)(output_node) # Contrast contrast_factor = utils.add_to_hp(self.contrast_factor, hp) if contrast_factor not in [0, (0, 0)]: output_node = layers.RandomContrast(contrast_factor)(output_node) return output_node
def build(self, hp, inputs=None): inputs = nest.flatten(inputs) utils.validate_num_inputs(inputs, 1) input_node = inputs[0] output_node = input_node kernel_size = utils.add_to_hp(self.kernel_size, hp) separable = self.separable if separable is None: separable = hp.Boolean("separable", default=False) if separable: conv = layer_utils.get_sep_conv(input_node.shape) else: conv = layer_utils.get_conv(input_node.shape) max_pooling = self.max_pooling if max_pooling is None: max_pooling = hp.Boolean("max_pooling", default=True) pool = layer_utils.get_max_pooling(input_node.shape) if self.dropout is not None: dropout = self.dropout else: dropout = hp.Choice("dropout", [0.0, 0.25, 0.5], default=0) for i in range(utils.add_to_hp(self.num_blocks, hp)): for j in range(utils.add_to_hp(self.num_layers, hp)): output_node = conv( utils.add_to_hp( self.filters, hp, "filters_{i}_{j}".format(i=i, j=j) ), kernel_size, padding=self._get_padding(kernel_size, output_node), activation="relu", )(output_node) if max_pooling: output_node = pool( kernel_size - 1, padding=self._get_padding(kernel_size - 1, output_node), )(output_node) if dropout > 0: output_node = layers.Dropout(dropout)(output_node) return output_node
def build(self, hp, inputs=None): input_tensor = nest.flatten(inputs)[0] tokenizer_layer = keras_layers.BertTokenizer( max_sequence_length=utils.add_to_hp(self.max_sequence_length, hp)) output_node = tokenizer_layer(input_tensor) bert_encoder = keras_layers.BertEncoder() output_node = bert_encoder(output_node) bert_encoder.load_pretrained_weights() return output_node