def __init__(self, french_window_size, french_vocab_size, english_window_size, english_vocab_size):
######vvv DO NOT CHANGE vvv##################
super(Transformer_Seq2Seq, self).__init__()
self.french_vocab_size = french_vocab_size # The size of the french vocab
self.english_vocab_size = english_vocab_size # The size of the english vocab
self.french_window_size = french_window_size # The french window size
self.english_window_size = english_window_size # The english window size
######^^^ DO NOT CHANGE ^^^##################
# Hyperparameters
self.optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
self.batch_size = 100
self.embedding_size = 30
# Define english and french embedding layers:
self.Embedding_English = tf.Variable(tf.random.truncated_normal(shape=[self.english_vocab_size, self.embedding_size], mean=0, stddev=0.01))
self.Embedding_French = tf.Variable(tf.random.truncated_normal(shape=[self.french_vocab_size, self.embedding_size], mean=0, stddev=0.01))
# Positional Encoder Layers
self.position_french = transformer.Position_Encoding_Layer(french_window_size, self.embedding_size)
self.position_english = transformer.Position_Encoding_Layer(english_window_size, self.embedding_size)
# Encoder and Decoder Layers:
self.encoder = transformer.Transformer_Block(self.embedding_size, is_decoder=False)
self.decoder = transformer.Transformer_Block(self.embedding_size, is_decoder=True)
# Dense Layer
self.dense = tf.keras.layers.Dense(self.english_vocab_size, activation='softmax')
def __init__(self, vocab_size, window_size, summary_window_size):
######vvv DO NOT CHANGE vvv##################
super(Transformer_Seq2Seq, self).__init__()
self.vocab_size = vocab_size
self.window_size = window_size
self.summary_window_size = summary_window_size
# 1) hyperparameters
self.encoder_size = 15
# Define batch size and optimizer/learning rate
self.batch_size = 100
self.embedding_size = 15
# Define paragraph and summaery embedding layers
self.embedding_layer = tf.keras.layers.Embedding(self.vocab_size, self.embedding_size, input_length=self.window_size)
self.summary_embedding_layer = tf.keras.layers.Embedding(self.vocab_size, self.embedding_size, input_length=self.summary_window_size)
# Create positional encoder layers
self.positional_layer = transformer.Position_Encoding_Layer(self.window_size, self.embedding_size)
self.summary_positional_layer = transformer.Position_Encoding_Layer(self.summary_window_size, self.embedding_size)
# Define encoder and decoder layers:
self.encoder_transformer = transformer.Transformer_Block(self.embedding_size, is_decoder=False, multi_headed=True)
self.decoder_transformer = transformer.Transformer_Block(self.embedding_size, is_decoder=True, multi_headed=True)
# Define dense layer(s)
self.dense = tf.keras.layers.Dense(self.vocab_size, activation='softmax')
def __init__(self, vocab_size):
super(Model, self).__init__()
self.vocab_size = vocab_size
self.window_size = 150
self.embedding_size = 100
self.batch_size = 64
self.E = tf.Variable(tf.random.truncated_normal(
[self.vocab_size, self.embedding_size],
dtype=tf.float32,
stddev=1e-1),
name='E')
self.gru_layer = tf.keras.layers.GRU(128,
return_sequences=False,
return_state=True)
# self.dense = tf.keras.layers.Dense(2, activation='softmax')
self.dense = tf.keras.layers.Dense(2, activation=None)
self.optimizer = tf.keras.optimizers.Adam(0.001)
self.position = transformer.Position_Encoding_Layer(
self.window_size, 100)
self.encoder = transformer.Transformer_Block(100,
is_decoder=False,
multi_headed=True)
def __init__(self, french_window_size, french_vocab_size,
english_window_size, english_vocab_size):
######vvv DO NOT CHANGE vvv##################
super(Transformer_Seq2Seq, self).__init__()
self.french_vocab_size = french_vocab_size # The size of the french vocab
self.english_vocab_size = english_vocab_size # The size of the english vocab
self.french_window_size = french_window_size # The french window size
self.english_window_size = english_window_size # The english window size
######^^^ DO NOT CHANGE ^^^##################
# TODO:
# 1) Define any hyperparameters
self.batch_size = 100
self.embedding_size = 128
self.optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
# 2) Define embeddings, encoder, decoder, and feed forward layers
# Define english and french embedding layers:
self.EE = tf.Variable(
tf.random.truncated_normal(
shape=[self.english_vocab_size, self.embedding_size],
mean=0,
stddev=0.01))
self.EF = tf.Variable(
tf.random.truncated_normal(
shape=[self.french_vocab_size, self.embedding_size],
mean=0,
stddev=0.01))
# Create positional encoder layers
self.position_fre = transformer.Position_Encoding_Layer(
french_window_size, self.embedding_size)
self.position_eng = transformer.Position_Encoding_Layer(
english_window_size, self.embedding_size)
# Define encoder and decoder layers:
self.encoder = transformer.Transformer_Block(self.embedding_size,
is_decoder=False)
self.decoder = transformer.Transformer_Block(self.embedding_size,
is_decoder=True)
# Define dense layer(s)
self.dense = tf.keras.layers.Dense(self.english_vocab_size,
activation='softmax')
def __init__(self, window_size, primary_vocab_size, ss_vocab_size):
super(Transformer_Seq2Seq, self).__init__()
self.primary_vocab_size = primary_vocab_size # The size of the primary vocab
self.ss_vocab_size = ss_vocab_size # The size of the ss vocab
self.window_size = window_size # The window size
self.learning_rate = 1e-2
self.optimizer = tf.keras.optimizers.Adam(self.learning_rate)
self.batch_size = 10
self.embedding_size = 32
# Define primary and ss embedding layers:
self.E_primary = tf.Variable(
tf.random.normal([self.primary_vocab_size, self.embedding_size],
stddev=.1))
self.E_ss = tf.Variable(
tf.random.normal([self.ss_vocab_size, self.embedding_size],
stddev=.1))
# Create positional encoder layers
self.pos_encoder_primary = transformer.Position_Encoding_Layer(
self.window_size, self.embedding_size)
self.pos_encoder_ss = transformer.Position_Encoding_Layer(
self.window_size, self.embedding_size)
# Define encoder and decoder layers:
self.encoder_1 = transformer.Transformer_Block(self.embedding_size,
False)
self.decoder_1 = transformer.Transformer_Block(self.embedding_size,
True)
self.decoder_2 = transformer.Transformer_Block(self.embedding_size,
True)
# Define dense layer(s)
self.dense = tf.keras.layers.Dense(self.ss_vocab_size,
activation='softmax')
def __init__(self, vocab_size, sentence_size):
super(Model, self).__init__()
# Model Vars
self.batch_size = 64
self.embedding_size = 34
self.vocab_size = vocab_size
self.window_size = sentence_size - 1
self.learning_rate = 0.01
self.optimizer = tf.keras.optimizers.Adam(learning_rate = self.learning_rate)
# Layers
self.embedding_model = tf.keras.layers.Embedding(self.vocab_size, self.embedding_size, input_length=self.window_size)
self.positional_model = transformer.Position_Encoding_Layer(self.window_size, self.embedding_size)
self.transformer1 = transformer.Transformer_Block(self.embedding_size, False)
# self.transformer2 = transformer.Transformer_Block(self.embedding_size, False)
self.dense_model = tf.keras.layers.Dense(self.vocab_size, activation="softmax")
def __init__(self, char_dict_len, isTrain):
super(Decoder, self).__init__()
if isTrain:
self.window_size = 8
else:
self.window_size = 1
self.isTrain = isTrain
self.dropout_rate = 0.1
self.decoder = transformer.Transformer_Block(_NUM_UNITS,
is_decoder=True,
multi_headed=True)
self.dense_layer3 = tf.keras.layers.Dense(input_dim=_NUM_UNITS,
units=char_dict_len)
self.dropout = tf.keras.layers.Dropout(self.dropout_rate)
def __init__(self, isTrain):
super(Encoder, self).__init__()
if isTrain:
self.window_size = 25
else:
self.window_size = 9
self.window_size = 25
self.isTrain = isTrain
self.dropout_rate = 0.1
self.pos_encoder_context = transformer.Position_Encoding_Layer(
self.window_size, _NUM_UNITS)
#transformer encoder
self.encoder = transformer.Transformer_Block(_NUM_UNITS,
is_decoder=False,
multi_headed=True)
self.dropout = tf.keras.layers.Dropout(self.dropout_rate)