def __init__(self, french_window_size, french_vocab_size, english_window_size, english_vocab_size):

		######vvv DO NOT CHANGE vvv##################
		super(Transformer_Seq2Seq, self).__init__()

		self.french_vocab_size = french_vocab_size # The size of the french vocab
		self.english_vocab_size = english_vocab_size # The size of the english vocab

		self.french_window_size = french_window_size # The french window size
		self.english_window_size = english_window_size # The english window size
		######^^^ DO NOT CHANGE ^^^##################

		# Hyperparameters
		self.optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
		self.batch_size = 100
		self.embedding_size = 30

		# Define english and french embedding layers:
		self.Embedding_English = tf.Variable(tf.random.truncated_normal(shape=[self.english_vocab_size, self.embedding_size], mean=0, stddev=0.01))
		self.Embedding_French = tf.Variable(tf.random.truncated_normal(shape=[self.french_vocab_size, self.embedding_size], mean=0, stddev=0.01))        
		
		# Positional Encoder Layers
		self.position_french = transformer.Position_Encoding_Layer(french_window_size, self.embedding_size)
		self.position_english = transformer.Position_Encoding_Layer(english_window_size, self.embedding_size)

		# Encoder and Decoder Layers:
		self.encoder = transformer.Transformer_Block(self.embedding_size, is_decoder=False)
		self.decoder = transformer.Transformer_Block(self.embedding_size, is_decoder=True)
            
		# Dense Layer
		self.dense = tf.keras.layers.Dense(self.english_vocab_size, activation='softmax')
Example #2
0
    def __init__(self, vocab_size, window_size, summary_window_size):

        ######vvv DO NOT CHANGE vvv##################
        super(Transformer_Seq2Seq, self).__init__()

        self.vocab_size = vocab_size
        self.window_size = window_size
        self.summary_window_size = summary_window_size

        # 1) hyperparameters
        self.encoder_size = 15
                
        # Define batch size and optimizer/learning rate
        self.batch_size = 100
        self.embedding_size = 15

        # Define paragraph and summaery embedding layers
        self.embedding_layer = tf.keras.layers.Embedding(self.vocab_size, self.embedding_size, input_length=self.window_size)
        self.summary_embedding_layer = tf.keras.layers.Embedding(self.vocab_size, self.embedding_size, input_length=self.summary_window_size)

        # Create positional encoder layers
        self.positional_layer = transformer.Position_Encoding_Layer(self.window_size, self.embedding_size)
        self.summary_positional_layer = transformer.Position_Encoding_Layer(self.summary_window_size, self.embedding_size)

        # Define encoder and decoder layers:
        self.encoder_transformer = transformer.Transformer_Block(self.embedding_size, is_decoder=False, multi_headed=True)
        self.decoder_transformer = transformer.Transformer_Block(self.embedding_size, is_decoder=True, multi_headed=True)

        # Define dense layer(s)
        self.dense = tf.keras.layers.Dense(self.vocab_size, activation='softmax')
    def __init__(self, vocab_size):

        super(Model, self).__init__()

        self.vocab_size = vocab_size
        self.window_size = 150
        self.embedding_size = 100
        self.batch_size = 64

        self.E = tf.Variable(tf.random.truncated_normal(
            [self.vocab_size, self.embedding_size],
            dtype=tf.float32,
            stddev=1e-1),
                             name='E')

        self.gru_layer = tf.keras.layers.GRU(128,
                                             return_sequences=False,
                                             return_state=True)
        # self.dense = tf.keras.layers.Dense(2, activation='softmax')
        self.dense = tf.keras.layers.Dense(2, activation=None)
        self.optimizer = tf.keras.optimizers.Adam(0.001)
        self.position = transformer.Position_Encoding_Layer(
            self.window_size, 100)
        self.encoder = transformer.Transformer_Block(100,
                                                     is_decoder=False,
                                                     multi_headed=True)
    def __init__(self, french_window_size, french_vocab_size,
                 english_window_size, english_vocab_size):
        ######vvv DO NOT CHANGE vvv##################
        super(Transformer_Seq2Seq, self).__init__()

        self.french_vocab_size = french_vocab_size  # The size of the french vocab
        self.english_vocab_size = english_vocab_size  # The size of the english vocab

        self.french_window_size = french_window_size  # The french window size
        self.english_window_size = english_window_size  # The english window size
        ######^^^ DO NOT CHANGE ^^^##################
        # TODO:
        # 1) Define any hyperparameters
        self.batch_size = 100
        self.embedding_size = 128
        self.optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)

        # 2) Define embeddings, encoder, decoder, and feed forward layers
        # Define english and french embedding layers:
        self.EE = tf.Variable(
            tf.random.truncated_normal(
                shape=[self.english_vocab_size, self.embedding_size],
                mean=0,
                stddev=0.01))
        self.EF = tf.Variable(
            tf.random.truncated_normal(
                shape=[self.french_vocab_size, self.embedding_size],
                mean=0,
                stddev=0.01))

        # Create positional encoder layers
        self.position_fre = transformer.Position_Encoding_Layer(
            french_window_size, self.embedding_size)
        self.position_eng = transformer.Position_Encoding_Layer(
            english_window_size, self.embedding_size)

        # Define encoder and decoder layers:
        self.encoder = transformer.Transformer_Block(self.embedding_size,
                                                     is_decoder=False)
        self.decoder = transformer.Transformer_Block(self.embedding_size,
                                                     is_decoder=True)

        # Define dense layer(s)
        self.dense = tf.keras.layers.Dense(self.english_vocab_size,
                                           activation='softmax')
Example #5
0
    def __init__(self, window_size, primary_vocab_size, ss_vocab_size):
        super(Transformer_Seq2Seq, self).__init__()

        self.primary_vocab_size = primary_vocab_size  # The size of the primary vocab
        self.ss_vocab_size = ss_vocab_size  # The size of the ss vocab

        self.window_size = window_size  # The window size

        self.learning_rate = 1e-2
        self.optimizer = tf.keras.optimizers.Adam(self.learning_rate)

        self.batch_size = 10
        self.embedding_size = 32

        # Define primary and ss embedding layers:
        self.E_primary = tf.Variable(
            tf.random.normal([self.primary_vocab_size, self.embedding_size],
                             stddev=.1))
        self.E_ss = tf.Variable(
            tf.random.normal([self.ss_vocab_size, self.embedding_size],
                             stddev=.1))

        # Create positional encoder layers
        self.pos_encoder_primary = transformer.Position_Encoding_Layer(
            self.window_size, self.embedding_size)
        self.pos_encoder_ss = transformer.Position_Encoding_Layer(
            self.window_size, self.embedding_size)

        # Define encoder and decoder layers:
        self.encoder_1 = transformer.Transformer_Block(self.embedding_size,
                                                       False)
        self.decoder_1 = transformer.Transformer_Block(self.embedding_size,
                                                       True)
        self.decoder_2 = transformer.Transformer_Block(self.embedding_size,
                                                       True)

        # Define dense layer(s)
        self.dense = tf.keras.layers.Dense(self.ss_vocab_size,
                                           activation='softmax')
Example #6
0
    def __init__(self, vocab_size, sentence_size):
        super(Model, self).__init__()

        # Model Vars
        self.batch_size = 64
        self.embedding_size = 34
        self.vocab_size = vocab_size
        self.window_size = sentence_size - 1
        self.learning_rate = 0.01
        self.optimizer = tf.keras.optimizers.Adam(learning_rate = self.learning_rate)

        # Layers
        self.embedding_model = tf.keras.layers.Embedding(self.vocab_size, self.embedding_size, input_length=self.window_size)
        self.positional_model = transformer.Position_Encoding_Layer(self.window_size, self.embedding_size)
        self.transformer1 = transformer.Transformer_Block(self.embedding_size, False)
        # self.transformer2 = transformer.Transformer_Block(self.embedding_size, False)
        self.dense_model = tf.keras.layers.Dense(self.vocab_size, activation="softmax")
Example #7
0
    def __init__(self, char_dict_len, isTrain):
        super(Decoder, self).__init__()

        if isTrain:
            self.window_size = 8
        else:
            self.window_size = 1

        self.isTrain = isTrain
        self.dropout_rate = 0.1

        self.decoder = transformer.Transformer_Block(_NUM_UNITS,
                                                     is_decoder=True,
                                                     multi_headed=True)

        self.dense_layer3 = tf.keras.layers.Dense(input_dim=_NUM_UNITS,
                                                  units=char_dict_len)

        self.dropout = tf.keras.layers.Dropout(self.dropout_rate)
Example #8
0
    def __init__(self, isTrain):
        super(Encoder, self).__init__()

        if isTrain:
            self.window_size = 25
        else:
            self.window_size = 9
        self.window_size = 25
        self.isTrain = isTrain
        self.dropout_rate = 0.1

        self.pos_encoder_context = transformer.Position_Encoding_Layer(
            self.window_size, _NUM_UNITS)

        #transformer encoder
        self.encoder = transformer.Transformer_Block(_NUM_UNITS,
                                                     is_decoder=False,
                                                     multi_headed=True)

        self.dropout = tf.keras.layers.Dropout(self.dropout_rate)