class SharedSeasonality(Seasonality):
"""Shared seasonality for time series."""
def __init__(self, seasonality_type="week", l2_reg=1e-3):
super().__init__(seasonality_type)
self.l2_reg = l2_reg
self._embedding = None
self._inputs = None
self._encoder = None
self._decoder = None
self._create_embedding()
self._create_inputs()
self._create_decoder()
self._create_encoder()
def _create_embedding(self):
self._embedding = Embedding(
self.seasonal_period,
output_dim=1,
name="seas_emb_%s" % self.seasonality_type,
embeddings_initializer=constant(0),
embeddings_regularizer=l1_l2(
l2=self.l2_reg) if self.l2_reg else None)
def _create_inputs(self):
self._inputs = [
Input(shape=(None, ),
name="inp_X_seas_%s" % self.seasonality_type),
Input(shape=(None, ), name="inp_Y_seas_%s" % self.seasonality_type)
]
def _create_decoder(self):
self._decoder = self._embedding(self._inputs[0]) + 1
def _create_encoder(self):
self._encoder = self._embedding(self._inputs[1]) + 1
def get_fit_inputs(self, ids, x):
return [x[:, :-1], x]
def get_predict_inputs(self, ids, x, last_date, increment):
# Expand x by one for predicting one step ahead
x_expanded = np.hstack([x, self._get_next_x(last_date, increment, 1)])
return [x, x_expanded]
def get_oos_predictions(self, first_oos_prediction, last_date, increment,
n_oos_steps):
x_oos = self._get_next_x(last_date, increment, n_oos_steps)
deseason = self._embedding.get_weights()[0][x_oos[:, 0]] + 1
oos_season = self._embedding.get_weights()[0][x_oos[:, 1:]][:, :,
0] + 1
return first_oos_prediction * oos_season / deseason
def get_weights(self):
return self._embedding.get_weights()[0] + 1
def model(words_per_sequence,
vocab_size,
embedding_weights_path="weights/embedding.npy"):
embedding_weights = embedding.load(embedding_weights_path).squeeze()
print(embedding_weights.shape)
embedding_layer = Embedding(embedding_weights.shape[0],
embedding_weights.shape[1],
input_length=words_per_sequence,
weights=[embedding_weights])
print(np.array(embedding_layer.get_weights()).squeeze().shape)
""" Calling embedding_layer.set_weights has been decreed a sin.
You have to use an undocumented weights argument. The weights should be
wrapped in a list as anything else would be against God """
#embedding_layer.set_weights(embedding_weights)
layer_input = Input((words_per_sequence, ))
x = embedding_layer(layer_input)
print(x.shape)
x = LSTM(embedding_weights.shape[-1], return_sequences=True)(x)
print(x.shape)
x = LSTM(embedding_weights.shape[-1], return_sequences=True)(x)
print(x.shape)
x = Dropout(0.5)(x)
print(x.shape)
x = TimeDistributed(Dense(vocab_size))(x)
print(x.shape)
x = Activation("softmax", name="softmax")(x)
print(x.shape)
return keras.Model(inputs=[layer_input], outputs=[x])
def define_generator(latent_dim, n_classes=3):
print("********** ENTERED generator *****************")
##### foundation for labels
in_label = Input(shape=(1, ))
embedding_layer = Embedding(n_classes, 8)
embedding_layer.trainable = True
li = embedding_layer(in_label)
n_nodes = 5 * 5
li = Dense(n_nodes)(li)
li = Reshape((5, 5, 1))(li)
print("generator... n_nodes, li.shape: ", n_nodes, li.shape)
##### foundation for 5x5 image
in_lat = Input(shape=(latent_dim, ))
n_nodes = 128 * 5 * 5
genX = Dense(n_nodes)(in_lat)
genX = LeakyReLU(alpha=0.2)(genX)
genX = Reshape((5, 5, 128))(genX)
dropout = 0.1
print("genX.shape: ", genX.shape)
##### merge image gen and label input
merge = Concatenate()([genX, li])
print("merge.shape: ", merge.shape)
##### create merged model
# upsample to 10x10
gen = Conv2DTranspose(128, (4, 4), strides=(2, 2), padding='same')(merge)
print("gen after CV2DT.shape: ", gen.shape)
gen = LeakyReLU(alpha=0.2)(gen)
gen = Dropout(dropout)(gen)
print("gen.shape: ", gen.shape)
# upsample to 20x20
gen = Conv2DTranspose(128, (4, 4), strides=(2, 2), padding='same')(gen)
gen = LeakyReLU(alpha=0.2)(gen)
print("gen.shape: ", gen.shape)
# upsample to 40x40
gen = Conv2DTranspose(128, (4, 4), strides=(2, 2), padding='same')(gen)
gen = LeakyReLU(alpha=0.2)(gen)
print("gen.shape: ", gen.shape)
# upsample to 80x80
gen = Conv2DTranspose(128, (4, 4), strides=(2, 2), padding='same')(gen)
gen = LeakyReLU(alpha=0.2)(gen)
print("gen.shape: ", gen.shape)
# output layer 80x80x3
out_layer = Conv2D(3, (5, 5), activation='tanh', padding='same')(gen)
print("out_layer.shape: ", out_layer.shape)
# define model
model = Model(inputs=[in_lat, in_label], outputs=out_layer)
opt = Adamax(lr=0.0002, beta_1=0.5, beta_2=0.999, epsilon=10e-8)
model.compile(loss=['binary_crossentropy'], optimizer=opt)
print("\nembedding_layer.get_weights(): \n", embedding_layer.get_weights())
model.summary()
plot_model(model, to_file='cgan/generator_model.png')
return model
def embeddings(model):
dim = 50
e = Embedding(word_dimensions,
dim,
input_length=max_input_length,
trainable=False)
model.add(e)
id2word_map = id2word()
weights = e.get_weights()[0]
loaded = load_embedding_text(dim)
for i in range(word_dimensions):
w = id2word_map[i]
if w in loaded:
weights[i] = np.array(loaded[w])
e.set_weights([weights])
def _create_model_predictor(self, lr):
y = self.y.ravel()
# Initial values from statsmodels implementation
if self.seasonal:
l0_start = y[np.arange(y.size) % self.seasonal_period == 0].mean()
lead, lag = y[self.seasonal_period: 2 * self.seasonal_period], y[:self.seasonal_period]
if self.trend == "multiplicative":
b0_start = ((lead - lag) / self.seasonal_period).mean()
elif self.trend == "additive":
b0_start = np.exp((np.log(lead.mean()) - np.log(lag.mean())) / self.seasonal_period)
if self.seasonal == "multiplicative":
s0_start = y[:self.seasonal_period] / l0_start
elif self.seasonal == "additive":
s0_start = y[:self.seasonal_period] - l0_start
elif self.trend:
l0_start = y[0]
if self.trend == "multiplicative":
b0_start = y[1] / l0_start
elif self.trend == "additive":
b0_start = y[1] - l0_start
else:
l0_start = y[0]
inp_y = Input(shape=(None, 1))
inp_emb_id = Input(shape=(1,)) # Dummy ID for embeddings
# (Ab)using embeddings here for initial value variables
init_l0 = [Embedding(1, 1, embeddings_initializer=constant(l0_start), name="l0")(inp_emb_id)[:, 0, :]]
if self.trend:
init_b0 = [Embedding(1, 1, embeddings_initializer=constant(b0_start), name="b0")(inp_emb_id)[:, 0, :]]
else:
init_b0 = []
if self.seasonal:
init_seas_emb = Embedding(1, self.seasonal_period, embeddings_initializer=RandomUniform(0.8, 1.2),
name="s0")
init_seas = [init_seas_emb(inp_emb_id)[:, 0, :]]
else:
init_seas = []
rnncell = ESRNN(self.trend, self.seasonal, self.seasonal_period)
rnn = RNN(rnncell, return_sequences=True, return_state=True)
out_rnn = rnn(inp_y, initial_state=init_l0 + init_b0 + init_seas)
if self.trend == "multiplicative":
l0_b0 = init_l0[0] * init_b0[0]
elif self.trend == "additive":
l0_b0 = init_l0[0] + init_b0[0]
else:
l0_b0 = init_l0[0]
if self.seasonal == "multiplicative":
l0_b0_s0 = l0_b0 * init_seas[0][:, :1]
elif self.seasonal == "additive":
l0_b0_s0 = l0_b0 + init_seas[0][:, :1]
else:
l0_b0_s0 = l0_b0
out = tf.keras.layers.concatenate([
tf.math.reduce_sum(l0_b0_s0, axis=1)[:, None, None],
out_rnn[0]
], axis=1)
model = Model(inputs=[inp_y, inp_emb_id], outputs=out)
model.compile(Adam(lr), "mse")
# predictor also outputs final state for predicting out-of-sample
predictor = Model(inputs=[inp_y, inp_emb_id], outputs=[out, out_rnn[1:]])
# Assign initial seasonality weights
if self.seasonal:
init_seas_emb.set_weights([s0_start.reshape(init_seas_emb.get_weights()[0].shape)])
return model, predictor
def define_discriminator(in_shape=(80, 80, 3), n_classes=3):
print("********** ENTERED discriminator *****************")
##### foundation for labels
in_label = Input(shape=(1, ))
embedding_layer = Embedding(n_classes, 8)
# embedding_layer.trainable = False
li = embedding_layer(in_label)
n_nodes = in_shape[0] * in_shape[1]
print(">>embedding>> in_shape[0], in_shape[1], n_nodes: ", in_shape[0],
in_shape[1], n_nodes)
li = Dense(n_nodes)(li)
li = Reshape((in_shape[0], in_shape[1], 1))(li)
# image input
dropout = 0.1
in_image = Input(shape=in_shape)
print("\nin_image: ", in_image)
# concat label as a channel
merge = Concatenate()([in_image, li])
print("\nmerge.shape: ", merge.shape)
# sample to 80x80
fe = Conv2D(128, (5, 5), padding='same')(merge)
fe = LeakyReLU(alpha=0.2)(fe)
fe = Dropout(dropout)(fe)
print("fe.shape: ", fe.shape)
# downsample to 40x40
fe = Conv2D(128, (5, 5), strides=(2, 2), padding='same')(fe)
fe = LeakyReLU(alpha=0.2)(fe)
# fe = Dropout(dropout)(fe)
print("fe.shape: ", fe.shape)
# downsample to 20x20
fe = Conv2D(128, (5, 5), strides=(2, 2), padding='same')(fe)
fe = LeakyReLU(alpha=0.2)(fe)
# fe = Dropout(dropout)(fe)
print("fe.shape: ", fe.shape)
# downsample to 10x10
fe = Conv2D(128, (5, 5), strides=(2, 2), padding='same')(fe)
fe = LeakyReLU(alpha=0.2)(fe)
# fe = Dropout(dropout)(fe)
print("fe.shape: ", fe.shape)
# downsample to 5x5
fe = Conv2D(128, (5, 5), strides=(2, 2), padding='same')(fe)
fe = LeakyReLU(alpha=0.2)(fe)
# fe = Dropout(dropout)(fe)
print("fe.shape: ", fe.shape)
# flatten feature maps
fe = Flatten()(fe)
# fe = Dropout(dropout)(fe)
print("fe flatten shape: ", fe.shape)
# output
out_layer = Dense(1, activation='sigmoid')(fe)
print("out_layer.shape: ", out_layer.shape)
# define model
model = Model([in_image, in_label], out_layer)
print("\nmodel: ", model)
# compile model
# opt = Adamax(lr=0.00007, beta_1=0.08, beta_2=0.999, epsilon=10e-8)
opt = Adamax(lr=0.00004, beta_1=0.08, beta_2=0.999, epsilon=10e-8)
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
print("\nembedding_layer.get_weights(): \n", embedding_layer.get_weights())
model.summary()
plot_model(model, to_file='cgan/discriminator_model.png')
return model
class NLPModel:
"""
The class representing the CNN model that learns the embedding of a specific personality trait.
"""
def __init__(
self,
train_inputs,
train_outputs,
weights=None,
voc_dim=60000,
features_number=200,
window_size=5,
filters_number=100,
hidden_units=50,
batch_size=32,
sentence_length=None,
train_zeros=False,
):
"""
The init method that creates the model. This model can learn an embedding from scratch or can tune a given embedding.
Parameters
----------
train_inputs: numpy.array
The numpy.array containing the encoded reviews of training set.
train_outputs: numpy.array
The numpy.array with len train_size, containing the reviews target.
weights: list, default: None
In the case of embedding tuning, this parameter represents the list, with shape (voc_dim, embedding_feature_number) representing the initial weights of the embedding to be tuned.
weights[i] must be the representation in the original embedding of term with index=i.
If weights is given, the model will tune the embedding.
voc_dim: int, default: 60000
In the case of embedding's learning from scratch, this parameter represents the vocabolary size.
features_number: int, default: 200
In the case of embedding's learning from scratch, this parameter represents the embedding's features number
window_size: int, default: 5
The windows dimension of convolution.
filters_number: int, default: 100
The number of convolution's filters.
hidden_units: int, default: 50
The number of units in the hidden layer.
batch_size: int, default: 32
The training's batch size.
sentence_length: int, default: None
The maximum length of a sentence. If none is set to the length of the longest sentence in training set + 20.
train_zeros: bool, default: False
True if you want to train the representation of padding tokens (tokens added to pad each review in such a way that all the reviews have the same lenght).
Parameters
----------
self.model: tensorflow.keras.models.Sequential
The model to be trained
self.train_inputs: numpy.array
The numpy.array containing the encoded reviews of training set.
self.train_outputs: numpy.array
The numpy.array with len train_size, containing the reviews target.
self.embedding_layer: tensorflow.keras.layers.Embedding
The model's embedding layer.
self.conv_layer: tensorflow.keras.layers.Conv2D
The model's convolutional layer.
self.pool_layer: tensorflow.keras.layers.MaxPooling2D
The model's max pool layer.
self.hidden_layer: tensorflow.keras.layers.Dense
The model's hidden layer before output layer.
self.output_layer: tensorflow.keras.layers.Dense
The model's output layer.
"""
self.voc_dim = voc_dim
self.features_number = features_number
self.window_size = window_size
self.filters_number = filters_number
self.hidden_units = hidden_units
self.batch_size = batch_size
self.train_zeros = train_zeros
if sentence_length is not None:
self.sentence_length = sentence_length
else:
self.sentence_length = self._maxLength(train_inputs)
self.train_inputs = self._createInputs(train_inputs)
assert train_outputs is not None
self.train_outputs = self._createOutputs(train_outputs,
train_outputs.shape[0])
if weights is not None:
self.weights = np.asarray(weights)
self.voc_dim = self.weights.shape[0]
self.features_number = self.weights.shape[1]
else:
self.weights = np.random.randint(low=0,
high=100,
size=(self.voc_dim,
self.features_number))
self.weights = self.weights / 100
self._initializeModel()
def _initializeModel(self):
self._createModel()
self._compileModel()
def _maxLength(self, inputs):
max_l = 0
for d in inputs:
if len(d) > max_l:
max_l = len(d)
return max_l + 20
def _createOutputs(self, x, number):
x = np.asarray(x)
return x.reshape(number, 1, 1, 1)
def _createInputs(self, inp):
return pad_sequences(inp,
maxlen=self.sentence_length,
padding="post",
truncating="post",
value=0)
def _createModel(self):
if self.train_zeros:
self._createModel_train_zeros()
else:
self._createModel_no_train_zeros()
def _createModel_train_zeros(self):
self.model = Sequential()
self.embedding_layer = Embedding(input_dim=self.voc_dim,
output_dim=self.features_number,
name="emb")
self.embedding_layer.build((None, ))
self.embedding_layer.set_weights([self.weights])
self.model.add(self.embedding_layer)
self.model.add(
Lambda(lambda t: t[..., None])
) # modifica della shape in modo che sia 4d, come richiesto da conv2d
self.conv_layer = Conv2D(
filters=self.filters_number,
kernel_size=(self.window_size, self.features_number),
strides=1,
padding="valid",
name="conv",
)
self.model.add(self.conv_layer)
self.pool_layer = MaxPooling2D(pool_size=(self.sentence_length -
self.window_size + 1, 1),
name="pool")
self.model.add(self.pool_layer)
self.hidden_layer = Dense(
self.hidden_units,
input_dim=self.filters_number,
activation=tf.nn.relu,
name="dense",
)
self.model.add(self.hidden_layer)
self.output_layer = Dense(1, activation="linear", name="output")
self.model.add(self.output_layer)
def _createModel_no_train_zeros(self):
self.input_layer = Input(shape=(self.sentence_length, ), name="input")
self.embedding_layer = Embedding(input_dim=self.voc_dim,
output_dim=self.features_number,
name="emb")
self.embedding_layer.build((None, ))
self.embedding_layer.set_weights([self.weights])
self.layers_inputs = (self.embedding_layer)(self.input_layer)
self.lambda_not_equal = Lambda(self._not_equal,
name="lambda_not_equal")
self.layers = (self.lambda_not_equal)(self.layers_inputs)
self.lambda_layer = Lambda(
lambda t: t[..., None], name="lambda_shape"
) # modifica della shape in modo che sia 4d, come richiesto da conv2d
self.layers = (self.lambda_layer)(self.layers)
self.conv_layer = Conv2D(
filters=self.filters_number,
kernel_size=(self.window_size, self.features_number),
strides=1,
padding="valid",
name="conv",
)
self.layers = (self.conv_layer)(self.layers)
self.pool_layer = MaxPooling2D(pool_size=(self.sentence_length -
self.window_size + 1, 1),
name="pool")
self.layers = (self.pool_layer)(self.layers)
self.hidden_layer = Dense(
self.hidden_units,
input_dim=self.filters_number,
activation=tf.nn.relu,
name="dense",
)
self.layers = (self.hidden_layer)(self.layers)
self.output_layer = Dense(1, activation="linear", name="output")
self.layers = (self.output_layer)(self.layers)
self.model = tf.keras.models.Model(inputs=self.input_layer,
outputs=self.layers)
def _not_equal(self, x):
zeros = tf.constant(0,
shape=(self.features_number, ),
dtype=np.float32)
not_equal = tf.dtypes.cast(M.not_equal(x, zeros), dtype=np.float32)
return x * not_equal
def _compileModel(self):
self.model.compile(optimizer="adagrad", loss="mse", metrics=["mse"])
for layer in self.model.layers:
print(layer.name, end=" ")
print(layer.output_shape)
def _fit_predict_train_zeros(self, x, y, root=None, epochs_number=10):
x = self._createInputs(x)
y_mse = y
y = self._createOutputs(y, x.shape[0])
self.predictions = []
self.mse = []
self.weights = []
for i in range(0, epochs_number):
print("\n________\nEPOCH ", i + 1, "/", epochs_number)
self.model.fit(
x=self.train_inputs,
y=self.train_outputs,
epochs=1,
batch_size=self.batch_size,
)
self.weights.append(self.embedding_layer.get_weights())
pred = self.model.predict(x)
pred = pred.reshape(pred.shape[0])
self.predictions.append(pred)
mse = sklearn.metrics.mean_squared_error(y_mse, pred)
self.mse.append(mse)
print("\nTEST RESULTS:\nMSE\n", mse)
if root is not None:
with open(os.path.join(root, "mse.pickle"), "wb") as f:
pickle.dump(self.mse, f)
with open(os.path.join(root, "weights.pickle"), "wb") as f:
pickle.dump(self.weights, f)
def _fit_predict_no_train_zeros(self, x, y, root=None, epochs_number=10):
x = self._createInputs(x)
y_mse = y
# TODO capire se serve questo y o posso cancellarlo
y = self._createOutputs(y, x.shape[0])
self.predictions = []
self.mse = []
self.weights = []
for i in range(0, epochs_number):
print("\n________\nEPOCH ", i + 1, "/", epochs_number)
self.model.fit(
x=self.train_inputs,
y=self.train_outputs,
epochs=1,
batch_size=self.batch_size,
)
self.weights.append(self.embedding_layer.get_weights())
pred = self.model.predict(x)
pred = pred.reshape(pred.shape[0])
self.predictions.append(pred)
mse = sklearn.metrics.mean_squared_error(y_mse, pred)
self.mse.append(mse)
print("\nTEST RESULTS:\nMSE\n", mse)
if root is not None:
with open(os.path.join(root, "mse.pickle"), "wb") as f:
pickle.dump(self.mse, f)
with open(os.path.join(root, "weights.pickle"), "wb") as f:
pickle.dump(self.weights, f)
def fit_predict(self,
test_inputs,
test_outputs,
root_path=None,
epochs_number=10):
"""
Fit the model on the training set and, at the end of each epoch, evaluate R2 and MSE metrics on test set.
Store performances and model's weights in the specific path.
Parameters
----------
test_inputs: numpy.array
the numpy.array containing the encoded reviews of test set.
test_outputs: numpy.array
the numpy.array with len test_size, containing the reviews target.
root_path: path, default: None
the path in which store weights and metrics
epochs_number: int, default: 10
train epochs' number.
Parameters
-------
self.predictions: list
The list containing model's predictions on test set after each epochs.
self.r2: list
The list containing model's predictions' estimated R2 on test set after each training epochs.
self.mse: list
The list containing model's predictions' estimated MSE on test set after each training epochs.
self.weights: list
The list containing model's weights after each training epochs.
"""
if root_path is not None:
create_dir(root_path)
if self.train_zeros:
self._fit_predict_train_zeros(test_inputs, test_outputs, root_path,
epochs_number)
else:
self._fit_predict_no_train_zeros(test_inputs, test_outputs,
root_path, epochs_number)
class FactorizedSeasonality(Seasonality):
def __init__(self,
n_time_series,
n_dim=1,
seasonality_type="weekday",
l2_reg=1e-3):
super().__init__(seasonality_type)
self.n_time_series = n_time_series
self.n_dim = n_dim
self.l2_reg = l2_reg
self._embedding = None
self._inputs = None
self._encoder = None
self._decoder = None
self._create_embedding()
self._create_inputs()
self._create_decoder()
self._create_encoder()
def _create_embedding(self):
# First embedding for time series ID
self._embedding = Embedding(
self.n_time_series,
output_dim=self.n_dim,
name="seas_emb_%s" % self.seasonality_type,
embeddings_initializer=constant(0),
embeddings_regularizer=l1_l2(
l2=self.l2_reg) if self.l2_reg else None)
self._seasonality_weights = Dense(self.seasonal_period,
activation='linear')
def _create_inputs(self):
self._inputs = [
Input(shape=(1, ),
name="timeseries_id_%s" % self.seasonality_type),
Input(shape=(None, ),
name="inp_X_seas_%s" % self.seasonality_type,
dtype=tf.int32),
Input(shape=(None, ),
name="inp_Y_seas_%s" % self.seasonality_type,
dtype=tf.int32)
]
def _create_decoder(self):
id_emb = self._embedding(self._inputs[0])[:, 0, :]
id_seas_values = self._seasonality_weights(id_emb)
seas_values = tf.gather(id_seas_values,
self._inputs[1],
axis=1,
batch_dims=-1)
self._decoder = seas_values[:, :, None] + 1
def _create_encoder(self):
id_emb = self._embedding(self._inputs[0])[:, 0, :]
id_seas_values = self._seasonality_weights(id_emb)
seas_values = tf.gather(id_seas_values,
self._inputs[2],
axis=1,
batch_dims=-1)
self._encoder = seas_values[:, :, None] + 1
def get_fit_inputs(self, ids, x):
return [ids, x[:, :-1], x]
def get_predict_inputs(self, ids, x, last_date, increment):
# Expand x by one for predicting one step ahead
x_expanded = np.hstack([x, self._get_next_x(last_date, increment, 1)])
return [ids, x, x_expanded]
def get_oos_predictions(self, first_oos_prediction, last_date, increment,
n_oos_steps):
x_oos = self._get_next_x(last_date, increment, n_oos_steps)
id_seas_values = self.get_weights()
deseason = np.take_along_axis(id_seas_values, x_oos[:, :1], axis=1)
oos_season = np.take_along_axis(id_seas_values, x_oos[:, 1:], axis=1)
return first_oos_prediction * oos_season / deseason
def get_weights(self):
return self._embedding.get_weights()[0] @ self._seasonality_weights.get_weights()[0] + \
self._seasonality_weights.get_weights()[1] + 1
class NLPModel:
def __init__(self, configs):
if configs.xavier_initializer:
embeddings_initializer = 'glorot_normal'
else:
embeddings_initializer = 'uniform'
self.vocabulary_length = configs.vocabulary_length
self.position_encode = positional_encoding(configs.embedding_size,
configs.max_sequence_length)
self.embedding = Embedding(
configs.vocabulary_length,
configs.embedding_size,
embeddings_initializer=embeddings_initializer)
self.encoder = Encoder(dim_input=configs.embedding_size,
model_hidden_size=configs.model_hidden_size,
ffn_hidden_size=configs.ffn_hidden_size,
heads=configs.attention_head_size,
num_layers=configs.layer_size)
self.decoder = Decoder(dim_input=configs.embedding_size,
dim_output=configs.vocabulary_length,
model_hidden_size=configs.model_hidden_size,
ffn_hidden_size=configs.ffn_hidden_size,
heads=configs.attention_head_size,
num_layers=configs.layer_size)
self.accuracy = tf.metrics.Accuracy()
self.optimizer = tf.optimizers.Adam(configs.learning_rate)
self._initialize(configs)
def _initialize(self, configs):
feature_temp = tf.zeros([1, configs.max_sequence_length],
dtype=tf.float32)
embed_temp = self.embedding(feature_temp)
enc_temp = self.encoder(embed_temp)
_ = self.decoder(embed_temp, enc_temp)
def train(self, features, labels):
with tf.GradientTape() as tape:
x_embed = self.embedding(features['input']) + self.position_encode
y_embed = self.embedding(features['output']) + self.position_encode
encoder_output = self.encoder(x_embed)
logits = self.decoder(y_embed, encoder_output)
predict = tf.argmax(logits, 2)
labels_ = tf.one_hot(labels, self.vocabulary_length)
# var_lists = self.encoder.trainable_variables + self.decoder.trainable_variables
var_lists = self.embedding.trainable_variables + self.encoder.trainable_variables + self.decoder.trainable_variables
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=labels_))
grad = tape.gradient(loss, var_lists)
self.optimizer.apply_gradients(zip(grad, var_lists))
self.accuracy.update_state(labels, predict)
def predict(self, feature):
x_embed = self.embedding(feature['input']) + self.position_encode
y_embed = self.embedding(feature['output']) + self.position_encode
encoder_output = self.encoder(x_embed)
logits = self.decoder(y_embed, encoder_output)
predict = tf.argmax(logits, 2)
return predict
def save_model(self, f_name):
w_dict = {}
w_dict['embedding'] = self.embedding.get_weights()
w_dict['encoder'] = self.encoder.get_weights()
w_dict['decoder'] = self.decoder.get_weights()
# f_name = os.path.join(model_path, model_name)
with open(f_name, 'wb') as f:
pickle.dump(w_dict, f)
print("model saved. (path: {})".format(f_name))
def load_model(self, f_name):
# f_name = os.path.join(model_path, model_name)
with open(f_name, 'rb') as f:
w_dict = pickle.load(f)
self.embedding.set_weights(w_dict['embedding'])
self.encoder.set_weights(w_dict['encoder'])
self.decoder.set_weights(w_dict['decoder'])
print("model loaded. (path: {})".format(f_name))