def __encoder(self, emb_inputs, mask): transformer_enc_layer = TransformerBlock( name='transformer_enc', num_heads=self.num_heads, residual_dropout=self.residual_dropout, attention_dropout=self.attention_dropout, compression_window_size=self.compression_window_size, use_masking=False, vanilla_wiring=True) coordinate_embedding_layer = TransformerCoordinateEmbedding( name="coordinate_emb1", max_transformer_depth=self.max_depth) transformer_act_layer = TransformerACT( name='adaptive_computation_time1') next_step_input = emb_inputs act_output = next_step_input for step in range(self.max_depth): next_step_input = coordinate_embedding_layer(next_step_input, step=step) next_step_input = transformer_enc_layer(next_step_input, padding_mask=mask) next_step_input, act_output = transformer_act_layer( next_step_input) transformer_act_layer.finalize() next_step_input = act_output return next_step_input
def universal_transformer_gpt_model( max_seq_length: int, vocabulary_size: int, word_embedding_size: int, transformer_depth: int, num_heads: int, transformer_dropout: float = 0.1, embedding_dropout: float = 0.6, l2_reg_penalty: float = 1e-6, confidence_penalty_weight: float = 0.1): """ A model which is similar to the one described by OpenAI in paper "Improving Language Understanding by Generative Pre-Training", except that it relies L2 regularization of the word embedding matrix (instead of the dropout), and uses Universal Transformer architecture. """ word_ids = Input(shape=(max_seq_length,), dtype='int32', name='word_ids') l2_regularizer = (regularizers.l2(l2_reg_penalty) if l2_reg_penalty else None) embedding_layer = ReusableEmbedding( vocabulary_size, word_embedding_size, input_length=max_seq_length, name='bpe_embeddings', # Regularization is based on paper "A Comparative Study on # Regularization Strategies for Embedding-based Neural Networks" # https://arxiv.org/pdf/1508.03721.pdf embeddings_regularizer=l2_regularizer) output_layer = TiedOutputEmbedding( projection_regularizer=l2_regularizer, projection_dropout=embedding_dropout, name='word_prediction_logits') coordinate_embedding_layer = TransformerCoordinateEmbedding( transformer_depth, name='coordinate_embedding') transformer_act_layer = TransformerACT(name='adaptive_computation_time') transformer_block = TransformerBlock( name='transformer', num_heads=num_heads, residual_dropout=transformer_dropout, attention_dropout=transformer_dropout, use_masking=True, vanilla_wiring=False) output_softmax_layer = Softmax(name='word_predictions') next_step_input, embedding_matrix = embedding_layer(word_ids) act_output = next_step_input for i in range(transformer_depth): next_step_input = coordinate_embedding_layer(next_step_input, step=i) next_step_input = transformer_block(next_step_input) next_step_input, act_output = transformer_act_layer(next_step_input) transformer_act_layer.finalize() next_step_input = act_output word_predictions = output_softmax_layer( output_layer([next_step_input, embedding_matrix])) model = Model(inputs=[word_ids], outputs=[word_predictions]) # Penalty for confidence of the output distribution, as described in # "Regularizing Neural Networks by Penalizing Confident # Output Distributions" (https://arxiv.org/abs/1701.06548) confidence_penalty = K.mean( confidence_penalty_weight * K.sum(word_predictions * K.log(word_predictions), axis=-1)) model.add_loss(confidence_penalty) return model
def make_transformer_model(max_seq_length: int, vocabulary_size: int, word_embedding_size: int, transformer_depth: int, num_heads: int, transformer_dropout: float = 0.1, embedding_dropout: float = 0.6, l2_reg_penalty: float = 1e-6, confidence_penalty_weight: float = 0.05): word_ids = Input(shape=(max_seq_length,), dtype='int32', name='word_ids') l2_regularizer = (regularizers.l2(l2_reg_penalty) if l2_reg_penalty else None) embedding_layer = ReusableEmbedding( vocabulary_size, word_embedding_size, input_length=max_seq_length, name='bpe_embeddings', # Regularization is based on paper "A Comparative Study on # Regularization Strategies for Embedding-based Neural Networks" # https://arxiv.org/pdf/1508.03721.pdf embeddings_regularizer=l2_regularizer) output_layer = TiedOutputEmbedding( projection_regularizer=l2_regularizer, projection_dropout=embedding_dropout, name='word_prediction_logits') coordinate_embedding_layer = TransformerCoordinateEmbedding( transformer_depth, name='coordinate_embedding') transformer_act_layer = TransformerACT(name='adaptive_computation_time') transformer_block = TransformerBlock( name='transformer', num_heads=num_heads, residual_dropout=transformer_dropout, attention_dropout=transformer_dropout, use_masking=True) output_softmax_layer = Softmax(name='word_predictions') next_step_input, embedding_matrix = embedding_layer(word_ids) act_output = next_step_input dropout_layer = Dropout(embedding_dropout, name='input_dropout') next_step_input = dropout_layer(next_step_input) for i in range(transformer_depth): next_step_input = coordinate_embedding_layer(next_step_input, step=i) next_step_input = transformer_block(next_step_input) next_step_input, act_output = transformer_act_layer(next_step_input) transformer_act_layer.finalize() next_step_input = act_output # depth_of_trainable_params(act_output) word_predictions = output_softmax_layer( output_layer([next_step_input, embedding_matrix])) model = Model(inputs=[word_ids], outputs=[word_predictions]) confidence_penalty = K.mean( confidence_penalty_weight * K.sum(word_predictions * K.log(word_predictions), axis=-1)) model.add_loss(confidence_penalty) return model
def get_transformer(transformer_input, transformer_depth=2): transformer_block = TransformerBlock( name='transformer', num_heads=8, residual_dropout=0.1, attention_dropout=0.1, use_masking=True) add_coordinate_embedding = TransformerCoordinateEmbedding( transformer_depth, name='coordinate_embedding') output = transformer_input # shape: (<batch size>, <sequence length>, <input size>) for step in range(transformer_depth): output = transformer_block( add_coordinate_embedding(output, step=step)) return output
def transformer_bert_model(max_seq_length: int, vocabulary_size: int, word_embedding_size: int, use_universal_transformer: bool, transformer_depth: int, num_heads: int, transformer_dropout: float = 0.1, embedding_dropout: float = 0.6, l2_reg_penalty: float = 1e-4): """ Builds a BERT-based model (Bidirectional Encoder Representations from Transformers) following paper "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding" (https://arxiv.org/abs/1810.04805) Depending on the value passed with `use_universal_transformer` argument, this function applies either an Adaptive Universal Transformer (2018) or a vanilla Transformer (2017) to do the job (the original paper uses vanilla Transformer). """ word_ids = Input(shape=(max_seq_length, ), dtype='int32', name='word_ids') segment_ids = Input(shape=(max_seq_length, ), dtype='int32', name='segment_ids') l2_regularizer = (regularizers.l2(l2_reg_penalty) if l2_reg_penalty else None) embedding_layer = ReusableEmbedding( vocabulary_size, word_embedding_size, input_length=max_seq_length, name='bpe_embeddings', # Regularization is based on paper "A Comparative Study on # Regularization Strategies for Embedding-based Neural Networks" # https://arxiv.org/pdf/1508.03721.pdf embeddings_regularizer=l2_regularizer) segment_embedding_layer = Embedding( 2, # "Segment A" and "Segment B" embeddings word_embedding_size, name='segment_embeddings') add_segment_layer = Add(name='add_segment') output_layer = TiedOutputEmbedding(projection_regularizer=l2_regularizer, projection_dropout=embedding_dropout, name='word_prediction_logits') output_softmax_layer = Softmax(name='word_predictions') coordinate_embedding_layer = TransformerCoordinateEmbedding( transformer_depth if use_universal_transformer else 1, name='coordinate_embedding') next_step_input, embedding_matrix = embedding_layer(word_ids) segment_embeddings = segment_embedding_layer(segment_ids) if use_universal_transformer: # Building a Universal Transformer (2018) act_layer = TransformerACT(name='adaptive_computation_time') transformer_block = TransformerBlock( name='transformer', num_heads=num_heads, residual_dropout=transformer_dropout, attention_dropout=transformer_dropout, # Allow bi-directional attention use_masking=False) act_output = next_step_input for i in range(transformer_depth): next_step_input = coordinate_embedding_layer(next_step_input, step=i) next_step_input = add_segment_layer( [next_step_input, segment_embeddings]) next_step_input = transformer_block(next_step_input) next_step_input, act_output = act_layer(next_step_input) act_layer.finalize() next_step_input = act_output else: # Building a Vanilla Transformer (described in # "Attention is all you need", 2017) next_step_input = coordinate_embedding_layer(next_step_input, step=0) next_step_input = add_segment_layer( [next_step_input, segment_embeddings]) for i in range(transformer_depth): next_step_input = ( TransformerBlock( name='transformer' + str(i), num_heads=num_heads, residual_dropout=transformer_dropout, attention_dropout=transformer_dropout, use_masking=False, # Allow bi-directional attention vanilla_wiring=True)(next_step_input)) word_predictions = output_softmax_layer( output_layer([next_step_input, embedding_matrix])) cls_node_slice = ( # selecting the first output position in each sequence # (responsible for classification) Lambda(lambda x: x[:, 0], name='cls_node_slicer')(next_step_input)) class_prediction = (Dense(1, name='class_prediction', activation='sigmoid')(cls_node_slice)) model = Model(inputs=[word_ids, segment_ids], outputs=[word_predictions, class_prediction]) return model
def build_model(max_length: int, embedding_matrix: Union[np.ndarray, Tuple[int]], transformer_depth: int, transformer_heads: int, filters: List[int], kernel_size: List[int], pool_size: List[int], conv_padding: str, pool_padding: str, dense_size: List[int], loaded_model: Optional[str] = None, fine_tune_model: bool = False, l2_penalty: Optional[float] = None, embedding_dropout: float = 0.6, transformer_dropout: float = 0.1, conv_dropout: float = 0.1, dense_dropout: Union[float, List[float]] = 0.3, classifier_dropout: float = 0.1, train_lm=True) -> Model: if not (len(filters) > 0 and len(kernel_size) > 0 and len(pool_size) > 0): logger.error( "There are no filters, kernel sizes or pool sizes specified for the CNN." ) raise ValueError( "There are no filters, kernel sizes or pool sizes specified for the CNN." ) if type(dense_dropout) != list: dense_dropout = [dense_dropout] if len(dense_size) > 0 and len(dense_size) != len(dense_dropout): max_list_length = max([len(dense_size), len(dense_dropout)]) new_dense_size = [] new_dense_dropout = [] for i in range(max_list_length): new_dense_size.append( dense_size[i] if i < len(dense_size) else dense_size[-1]) new_dense_dropout.append(dense_dropout[i] if i < len(dense_dropout) else dense_dropout[-1]) dense_size = new_dense_size dense_dropout = new_dense_dropout logger.warning( "Lists given for dense layer sizes and dense layer dropout rates are not the same length. " "The shorter lists are padded using the last value to match the length of the longest." ) if len(filters) != len(kernel_size) or len(filters) != len( pool_size) or len(kernel_size) != len(pool_size): max_list_length = max([len(filters), len(kernel_size), len(pool_size)]) new_filters = [] new_kernel_size = [] new_pool_size = [] for i in range(max_list_length): new_filters.append(filters[i] if i < len(filters) else filters[-1]) new_kernel_size.append( kernel_size[i] if i < len(kernel_size) else kernel_size[-1]) new_pool_size.append( pool_size[i] if i < len(pool_size) else pool_size[-1]) filters = new_filters kernel_size = new_kernel_size pool_size = new_pool_size logger.warning( "Lists given for convolutional filters, kernel sizes and pooling sizes had different lengths. " "The shorter lists are padded using the last value to match the length of the longest." ) original_model = None if loaded_model: # load the specified model original_model = load_model(loaded_model, custom_objects={ "perplexity": perplexity, "lm_accuracy": lm_accuracy }) # regularizer for embedding layer l2_regularizer = l2(l2_penalty) if l2_penalty else None # input encoded as integers raw_input = Input(shape=(max_length, ), name="input") # embedding layer, initialised with embedding matrix weights for now embedding_weights = [ original_model.get_layer(name="word_embedding").get_weights()[0] if loaded_model else embedding_matrix ] embedding_layer = ReusableEmbedding( input_dim=(embedding_matrix[0] if type(embedding_matrix) == tuple else embedding_matrix.shape[0]), output_dim=(embedding_matrix[1] if type(embedding_matrix) == tuple else embedding_matrix.shape[1]), input_length=max_length, name="word_embedding", weights=(None if type(embedding_matrix) == tuple and not loaded_model else embedding_weights), embeddings_regularizer=l2_regularizer) # "transpose" of embedding matrix to map back to vocabulary if loaded_model: output_weights = original_model.get_layer( name="word_prediction_logits").get_weights() output_layer = TiedOutputEmbedding( projection_regularizer=l2_regularizer, projection_dropout=embedding_dropout, name="word_prediction_logits", weights=output_weights) else: output_layer = TiedOutputEmbedding( projection_regularizer=l2_regularizer, projection_dropout=embedding_dropout, name="word_prediction_logits") # transformer as taken from here: https://github.com/kpot/keras-transformer/blob/master/example/models.py if loaded_model: position_weights = original_model.get_layer( name="position_embedding").get_weights() position_embedding = TransformerCoordinateEmbedding( max_transformer_depth=1, name="position_embedding", weights=position_weights) else: position_embedding = TransformerCoordinateEmbedding( max_transformer_depth=1, name="position_embedding") transformer_input, embedding_matrix = embedding_layer(raw_input) transformer_output = position_embedding(transformer_input, step=0) for i in range(transformer_depth): block_name = "transformer" + str(i) # define transformer block transformer_block = TransformerBlock( name=block_name, num_heads=transformer_heads, residual_dropout=transformer_dropout, attention_dropout=transformer_dropout, use_masking=True, vanilla_wiring=True) # build the layers in the block because apparently you have to do that if loaded_model: if i == 0: transformer_block.attention_layer.build( original_model.get_layer( "position_embedding").output_shape) else: transformer_block.attention_layer.build( original_model.get_layer( "transformer{}_normalization2".format(i - 1)).output_shape) transformer_block.norm1_layer.build( original_model.get_layer(block_name + "_self_attention").output_shape) transformer_block.norm2_layer.build( original_model.get_layer(block_name + "_normalization1").output_shape) transformer_block.transition_layer.build( original_model.get_layer(block_name + "_normalization1").output_shape) # set weights for all the contained layers manually transformer_block.attention_layer.set_weights( original_model.get_layer( name=(block_name + "_self_attention")).get_weights()) transformer_block.norm1_layer.set_weights( original_model.get_layer( name=(block_name + "_normalization1")).get_weights()) transformer_block.norm2_layer.set_weights( original_model.get_layer( name=(block_name + "_normalization2")).get_weights()) transformer_block.transition_layer.set_weights( original_model.get_layer(name=(block_name + "_transition")).get_weights()) # pass output of last layer through transformer transformer_output = transformer_block(transformer_output) # nothing special to load for softmax softmax_layer = Softmax(name="word_predictions") lm_output_logits = output_layer([transformer_output, embedding_matrix]) lm_output = softmax_layer(lm_output_logits) if not fine_tune_model: m = Model(inputs=raw_input, outputs=lm_output) return m loaded_layer_names = [] if loaded_model: loaded_layer_names = [layer.name for layer in original_model.layers] # convolution layer(s) conv_dropout = Dropout(conv_dropout, name="conv_dropout") conv_output = transformer_output for i in range(len(filters)): # construct and possibly load convolutional layer conv_layer_name = "conv_{}".format(i) convolution = Conv1D(filters[i], kernel_size[i], padding=conv_padding, activation="relu", name=conv_layer_name) if loaded_model and conv_layer_name in loaded_layer_names: layer = original_model.get_layer(name=conv_layer_name) convolution.build(layer.input_shape) convolution.set_weights(layer.get_weights()) # construct max pooling, no weights to load pooling = MaxPooling1D(pool_size[i], padding=pool_padding, name="max_pool_{}".format(i)) # get output/input of next layer conv_output = pooling(convolution(conv_dropout(conv_output))) # dense layer(s) flatten = Flatten(name="flatten") dense_output = flatten(conv_output) for i in range(len(dense_size)): # construct and possibly load dense layer dense_layer_name = "dense_{}".format(i) dense = Dense(dense_size[i], name=dense_layer_name) if loaded_model and dense_layer_name in loaded_layer_names: layer = original_model.get_layer(name=dense_layer_name) dense.build(layer.input_shape) dense.set_weights(layer.get_weights()) # nothing to load for dropout dropout = Dropout(rate=dense_dropout[i], name="dense_dropout_{}".format(i)) # get output dense_output = dense(dropout(dense_output)) # classification layer classifier_dropout = Dropout(classifier_dropout, name="classifier_dropout") classifier = Dense(1, name="classifier") classifier_prediction = Activation("sigmoid", name="classifier_prediction") classifier_output = classifier_prediction( classifier(classifier_dropout(dense_output))) if train_lm: m = Model(inputs=raw_input, outputs=[lm_output, classifier_output]) else: m = Model(inputs=raw_input, outputs=classifier_output) return m
def transformer_bert_model(max_seq_length: int, time_window_size: int, vocabulary_size: int, concept_embedding_size: int, depth: int, num_heads: int, transformer_dropout: float = 0.1, embedding_dropout: float = 0.6, l2_reg_penalty: float = 1e-4): """ Builds a BERT-based model (Bidirectional Encoder Representations from Transformers) following paper "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding" (https://arxiv.org/abs/1810.04805) Depending on the value passed with `use_universal_transformer` argument, this function applies either an Adaptive Universal Transformer (2018) or a vanilla Transformer (2017) to do the job (the original paper uses vanilla Transformer). """ masked_concept_ids = tf.keras.layers.Input(shape=(max_seq_length, ), dtype='int32', name='masked_concept_ids') concept_ids = tf.keras.layers.Input(shape=(max_seq_length, ), dtype='int32', name='concept_ids') time_stamps = tf.keras.layers.Input(shape=(max_seq_length, ), dtype='int32', name='time_stamps') mask = tf.keras.layers.Input(shape=(max_seq_length, ), dtype='int32', name='mask') concept_mask = tf.expand_dims(tf.expand_dims(mask, axis=1), axis=1) l2_regularizer = (tf.keras.regularizers.l2(l2_reg_penalty) if l2_reg_penalty else None) embedding_layer = ReusableEmbedding( vocabulary_size, concept_embedding_size, input_length=max_seq_length, name='bpe_embeddings', # Regularization is based on paper "A Comparative Study on # Regularization Strategies for Embedding-based Neural Networks" # https://arxiv.org/pdf/1508.03721.pdf embeddings_regularizer=l2_regularizer) time_embedding_layer = TimeSelfAttention(vocab_size=vocabulary_size, target_seq_len=max_seq_length, context_seq_len=max_seq_length, time_window_size=time_window_size, return_logits=True) encoder_layer = Encoder(name='encoder', num_layers=depth, d_model=concept_embedding_size, num_heads=num_heads, dropout_rate=transformer_dropout) output_layer = TiedOutputEmbedding(projection_regularizer=l2_regularizer, projection_dropout=embedding_dropout, name='concept_prediction_logits') softmax_layer = tf.keras.layers.Softmax(name='concept_predictions') coordinate_embedding_layer = TransformerCoordinateEmbedding( 1, name='coordinate_embedding') next_step_input, embedding_matrix = embedding_layer(masked_concept_ids) # Building a Vanilla Transformer (described in # "Attention is all you need", 2017) next_step_input = coordinate_embedding_layer(next_step_input, step=0) # shape = (batch_size, seq_len, seq_len) time_attention = time_embedding_layer([concept_ids, time_stamps, mask]) # pad a dimension to accommodate the head split time_attention = tf.expand_dims(time_attention, axis=1) next_step_input, _ = encoder_layer(next_step_input, concept_mask, time_attention) concept_predictions = softmax_layer( output_layer([next_step_input, embedding_matrix])) model = tf.keras.Model( inputs=[masked_concept_ids, concept_ids, time_stamps, mask], outputs=[concept_predictions]) return model
def vanilla_transformer_gpt_model( max_seq_length: int, vocabulary_size: int, word_embedding_size: int, transformer_depth: int, num_heads: int, transformer_dropout: float = 0.1, embedding_dropout: float = 0.6, l2_reg_penalty: float = 1e-6, confidence_penalty_weight: float = 0.1, agglomerative_attention: bool = False, use_convolutions: bool = False, use_coordinate_embeddings: bool = True, convolution_width: int = 0, dropout_cls: Type[Layer] = Dropout ): """ A model which is almost identical to the one described by OpenAI in paper "Improving Language Understanding by Generative Pre-Training", except that it uses L2 regularization of the word embedding matrix, instead of the dropout. """ word_ids = Input(shape=(max_seq_length,), dtype='int32', name='word_ids') l2_regularizer = (keras.regularizers.l2(l2_reg_penalty) if l2_reg_penalty else None) embedding_layer = ReusableEmbedding( vocabulary_size, word_embedding_size, input_length=max_seq_length, name='bpe_embeddings', # Regularization is based on paper "A Comparative Study on # Regularization Strategies for Embedding-based Neural Networks" # https://arxiv.org/pdf/1508.03721.pdf embeddings_regularizer=l2_regularizer) output_layer = TiedOutputEmbedding( projection_regularizer=l2_regularizer, projection_dropout=embedding_dropout, name='word_prediction_logits') conv_layer = keras.layers.Conv1D( word_embedding_size, convolution_width, padding='causal', activation='relu', kernel_initializer='he_uniform', name='convolution') coordinate_embedding_layer = TransformerCoordinateEmbedding( 1, name='coordinate_embedding') output_softmax_layer = Softmax(name='word_predictions') next_step_input, embedding_matrix = embedding_layer(word_ids) if use_convolutions: next_step_input = conv_layer(next_step_input) if use_coordinate_embeddings: next_step_input = coordinate_embedding_layer(next_step_input, step=0) for i in range(transformer_depth): next_step_input = ( TransformerBlock( name='transformer' + str(i), num_heads=num_heads, residual_dropout=transformer_dropout, attention_dropout=transformer_dropout, use_masking=True, vanilla_wiring=True, agglomerative_attention=agglomerative_attention, dropout_cls=dropout_cls, ) (next_step_input)) word_predictions = output_softmax_layer( output_layer([next_step_input, embedding_matrix])) model = keras.models.Model(inputs=[word_ids], outputs=[word_predictions]) # Penalty for confidence of the output distribution, as described in # "Regularizing Neural Networks by Penalizing Confident # Output Distributions" (https://arxiv.org/abs/1701.06548) confidence_penalty = K.mean( confidence_penalty_weight * K.sum(word_predictions * K.log(word_predictions), axis=-1)) model.add_loss(confidence_penalty) return model
def build_model(max_length: int, embedding_matrix: Union[np.ndarray, Tuple[int]], transformer_depth: int, transformer_heads: int, cell_type: str, cell_size: int, cell_stack_size: int, filters: List[int], kernel_size: List[int], pool_size: List[int], conv_padding: str, pool_padding: str, dense_size: List[int], loaded_model: Optional[str] = None, l2_penalty: Optional[float] = None, embedding_dropout: float = 0.6, transformer_dropout: float = 0.1, conv_dropout: float = 0.1, dense_dropout: Union[float, List[float]] = 0.3, classifier_dropout: float = 0.1, no_cnn: bool = False, train_lm: bool = True) -> Model: if type(dense_dropout) != list: dense_dropout = [dense_dropout] if len(dense_size) > 0 and len(dense_size) != len(dense_dropout): max_list_length = max([len(dense_size), len(dense_dropout)]) new_dense_size = [] new_dense_dropout = [] for i in range(max_list_length): new_dense_size.append( dense_size[i] if i < len(dense_size) else dense_size[-1]) new_dense_dropout.append(dense_dropout[i] if i < len(dense_dropout) else dense_dropout[-1]) dense_size = new_dense_size dense_dropout = new_dense_dropout logger.warning( "Lists given for dense layer sizes and dense layer dropout rates are not the same length. " "The shorter lists are padded using the last value to match the length of the longest." ) if not no_cnn and len(filters) != len(kernel_size) or len(filters) != len( pool_size) or len(kernel_size) != len(pool_size): max_list_length = max([len(filters), len(kernel_size), len(pool_size)]) new_filters = [] new_kernel_size = [] new_pool_size = [] for i in range(max_list_length): new_filters.append(filters[i] if i < len(filters) else filters[-1]) new_kernel_size.append( kernel_size[i] if i < len(kernel_size) else kernel_size[-1]) new_pool_size.append( pool_size[i] if i < len(pool_size) else pool_size[-1]) filters = new_filters kernel_size = new_kernel_size pool_size = new_pool_size logger.warning( "Lists given for convolutional filters, kernel sizes and pooling sizes had different lengths. " "The shorter lists are padded using the last value to match the length of the longest." ) cell_type_name = cell_type.lower() if cell_type_name == "lstm": cell_type = LSTM elif cell_type_name == "gru": cell_type = GRU # regularizer for embedding layer l2_regularizer = l2(l2_penalty) if l2_penalty else None # input encoded as integers raw_input = Input(shape=(max_length, ), name="input") # embedding layer, initialised with embedding matrix weights for now embedding_layer = ReusableEmbedding( input_dim=(embedding_matrix[0] if type(embedding_matrix) == tuple else embedding_matrix.shape[0]), output_dim=(embedding_matrix[1] if type(embedding_matrix) == tuple else embedding_matrix.shape[1]), input_length=max_length, name="word_embedding", weights=(None if type(embedding_matrix) == tuple else [embedding_matrix]), embeddings_regularizer=l2_regularizer) # "transpose" of embedding matrix to map back to vocabulary lm_output = TiedOutputEmbedding(projection_regularizer=l2_regularizer, projection_dropout=embedding_dropout, name="word_prediction_logits") # transformer as taken from here: https://github.com/kpot/keras-transformer/blob/master/example/models.py position_embedding = TransformerCoordinateEmbedding( max_transformer_depth=1, name="position_embedding") transformer_blocks = [] for i in range(transformer_depth): block_name = "transformer" + str(i) # define transformer block transformer_block = TransformerBlock( name=block_name, num_heads=transformer_heads, residual_dropout=transformer_dropout, attention_dropout=transformer_dropout, use_masking=True, vanilla_wiring=True) transformer_blocks.append(transformer_block) # nothing special to load for softmax lm_prediction = Softmax(name="word_predictions") # RNN rnn_cells = [] for i in range(cell_stack_size - 1): cell_name = "{}_{}".format(cell_type_name, i) cell = Bidirectional(cell_type(cell_size, return_sequences=True, name=cell_name), name="bidirectional_{}".format(cell_name)) rnn_cells.append(cell) cell_name = "{}_{}".format(cell_type_name, cell_stack_size - 1) cell = Bidirectional(cell_type(cell_size, return_sequences=(not no_cnn), name=cell_name), name="bidirectional_{}".format(cell_name)) rnn_cells.append(cell) # flattening for RNN/transformer outputs and CNN outputs flatten = Flatten(name="flatten") # convolution layer(s) conv_dropout = Dropout(conv_dropout, name="conv_dropout") conv_layers = [] for i in range(len(filters)): # construct and possibly load convolutional layer conv_layer_name = "conv_{}".format(i) convolution = Conv1D(filters[i], kernel_size[i], padding=conv_padding, activation="relu", name=conv_layer_name) # construct max pooling, no weights to load pooling = MaxPooling1D(pool_size[i], padding=pool_padding, name="max_pool_{}".format(i)) conv_layers.append((convolution, pooling)) # dense layer(s) dense_layers = [] for i in range(len(dense_size)): # nothing to load for dropout dropout = Dropout(rate=dense_dropout[i], name="dense_dropout_{}".format(i)) # construct and possibly load dense layer dense_layer_name = "dense_{}".format(i) dense = Dense(dense_size[i], name=dense_layer_name) # get output dense_layers.append((dropout, dense)) # classification layer classifier_dropout = Dropout(classifier_dropout, name="classifier_dropout") classifier = Dense(1, name="classifier") classifier_prediction = Activation("sigmoid", name="classifier_prediction") # embedding used for both parts of the model embedded, embedding_matrix = embedding_layer(raw_input) # BUILD THE ACTUAL MODEL # transformer transformer_output = position_embedding(embedded, step=0) for tb in transformer_blocks: transformer_output = tb(transformer_output) # language model loss if it is used lm_output = lm_output([transformer_output, embedding_matrix]) lm_output = lm_prediction(lm_output) # RNN rnn_output = embedded for rc in rnn_cells: rnn_output = rc(rnn_output) # handle concatenation of both outputs if no_cnn: # flatten both and concatenate them concat = Concatenate(name="concat_flattened") classifier_output = concat( [flatten(transformer_output), flatten(rnn_output)]) else: # concatenate the extracted features concat = Concatenate(name="concat_features") classifier_output = concat([transformer_output, rnn_output]) # pass through CNN for cl in conv_layers: classifier_output = cl[1](cl[0](conv_dropout(classifier_output))) # pass through dense layer(s) classifier_output = flatten(classifier_output) for dl in dense_layers: classifier_output = dl[1](dl[0](classifier_output)) # do classification classifier_output = classifier_prediction( classifier(classifier_dropout(classifier_output))) if train_lm: m = Model(inputs=raw_input, outputs=[lm_output, classifier_output]) else: m = Model(inputs=raw_input, outputs=classifier_output) return m
def build_model(max_length, loaded_model=None, fine_tune_model=False, embedding_matrix=None, transformer_depth=8, transformer_heads=8, l2_penalty=None, embedding_dropout=0.6, transformer_dropout=0.1, classifier_dropout=0.1, transformer_output_handling="flatten", print_info=False, train_lm=True): original_model = None if loaded_model: # load the specified model original_model = load_model(loaded_model, custom_objects={ "perplexity": perplexity, "lm_accuracy": lm_accuracy, "SeqSelfAttention": SeqSelfAttention, "ScaledDotProductAttention": ScaledDotProductAttention }) # regularizer for embedding layer l2_regularizer = l2(l2_penalty) if l2_penalty else None # input encoded as integers raw_input = Input(shape=(max_length, ), name="input") # embedding layer, initialised with embedding matrix weights for now embedding_weights = [ original_model.get_layer(name="word_embedding").get_weights()[0] if loaded_model else embedding_matrix ] embedding_layer = ReusableEmbedding( input_dim=(embedding_matrix[0] if type(embedding_matrix) == tuple else embedding_matrix.shape[0]), output_dim=(embedding_matrix[1] if type(embedding_matrix) == tuple else embedding_matrix.shape[1]), input_length=max_length, name="word_embedding", weights=(None if type(embedding_matrix) == tuple and not loaded_model else embedding_weights), embeddings_regularizer=l2_regularizer) # "transpose" of embedding matrix to map back to vocabulary if loaded_model: output_weights = original_model.get_layer( name="word_prediction_logits").get_weights() output_layer = TiedOutputEmbedding( projection_regularizer=l2_regularizer, projection_dropout=embedding_dropout, name="word_prediction_logits", weights=output_weights) else: output_layer = TiedOutputEmbedding( projection_regularizer=l2_regularizer, projection_dropout=embedding_dropout, name="word_prediction_logits") # transformer as taken from here: https://github.com/kpot/keras-transformer/blob/master/example/models.py if loaded_model: position_weights = original_model.get_layer( name="position_embedding").get_weights() position_embedding = TransformerCoordinateEmbedding( max_transformer_depth=1, name="position_embedding", weights=position_weights) else: position_embedding = TransformerCoordinateEmbedding( max_transformer_depth=1, name="position_embedding") transformer_input, embedding_matrix = embedding_layer(raw_input) transformer_output = position_embedding(transformer_input, step=0) for i in range(transformer_depth): block_name = "transformer" + str(i) # define transformer block transformer_block = TransformerBlock( name=block_name, num_heads=transformer_heads, residual_dropout=transformer_dropout, attention_dropout=transformer_dropout, use_masking=True, vanilla_wiring=True) # build the layers in the block because apparently you have to do that if loaded_model: if i == 0: transformer_block.attention_layer.build( original_model.get_layer( "position_embedding").output_shape) else: transformer_block.attention_layer.build( original_model.get_layer( "transformer{}_normalization2".format(i - 1)).output_shape) transformer_block.norm1_layer.build( original_model.get_layer(block_name + "_self_attention").output_shape) transformer_block.norm2_layer.build( original_model.get_layer(block_name + "_normalization1").output_shape) transformer_block.transition_layer.build( original_model.get_layer(block_name + "_normalization1").output_shape) # set weights for all the contained layers manually transformer_block.attention_layer.set_weights( original_model.get_layer( name=(block_name + "_self_attention")).get_weights()) transformer_block.norm1_layer.set_weights( original_model.get_layer( name=(block_name + "_normalization1")).get_weights()) transformer_block.norm2_layer.set_weights( original_model.get_layer( name=(block_name + "_normalization2")).get_weights()) transformer_block.transition_layer.set_weights( original_model.get_layer(name=(block_name + "_transition")).get_weights()) # pass output of last layer through transformer transformer_output = transformer_block(transformer_output) if print_info: logger.debug("transformer_output shape: {}".format( K.int_shape(transformer_output[0] if fine_tune_model else transformer_output))) # nothing special to load for softmax softmax_layer = Softmax(name="word_predictions") lm_output_logits = output_layer([transformer_output, embedding_matrix]) lm_output = softmax_layer(lm_output_logits) if print_info: logger.debug("lm_output_logits shape: {}".format( K.int_shape(lm_output_logits))) logger.debug("output shape: {}".format(K.int_shape(lm_output))) if not fine_tune_model: m = Model(inputs=raw_input, outputs=lm_output) return m loaded_layer_names = [] if loaded_model: loaded_layer_names = [layer.name for layer in original_model.layers] # for concatenation transformer outputs early flatten = Flatten(name="flatten_transformer_output") max_pooling = Lambda(lambda x: K.max(x, axis=1), name="max_pooling") mean_pooling = Lambda(lambda x: K.mean(x, axis=1), name="mean_pooling") self_attention = SeqSelfAttention(name="self_attention") scaled_dot_attention = ScaledDotProductAttention( name="scaled_dot_attention") dropout = Dropout(rate=classifier_dropout, name="classifier_dropout") options = { "flatten": flatten, "max_pooling": max_pooling, "mean_pooling": mean_pooling, "self_attention": self_attention, "scaled_dot_attention": scaled_dot_attention } dense = Dense(2, activation=None, name="dense") if loaded_model and "dense" in loaded_layer_names: layer = original_model.get_layer(name="dense") dense.build(layer.input_shape) dense.set_weights(layer.get_weights()) pooling_layer = options[transformer_output_handling] if loaded_model and transformer_output_handling in loaded_layer_names: layer = original_model.get_layer(name=transformer_output_handling) pooling_layer.build(layer.input_shape) pooling_layer.set_weights(layer.get_weights()) if "attention" in transformer_output_handling: handled_output = flatten(pooling_layer(transformer_output)) else: handled_output = pooling_layer(transformer_output) classifier_logits = dense(dropout(handled_output)) classifier_output = Softmax( name="classifier_prediction")(classifier_logits) if train_lm: m = Model(inputs=raw_input, outputs=[lm_output, classifier_output]) else: m = Model(inputs=raw_input, outputs=classifier_output) # m = Model(inputs=raw_input, outputs=lm_output) return m