class NNRF(GenericModel):
"""Non-incremental model role-filler
"""
def __init__(self,
n_word_vocab=50001,
n_role_vocab=7,
n_factors_emb=256,
n_factors_cls=512,
n_hidden=256,
word_vocabulary={},
role_vocabulary={},
unk_word_id=50000,
unk_role_id=7,
missing_word_id=50001,
using_dropout=False,
dropout_rate=0.3,
optimizer='adagrad',
loss='sparse_categorical_crossentropy',
metrics=['accuracy']):
super(NNRF, self).__init__(n_word_vocab, n_role_vocab, n_factors_emb,
n_hidden, word_vocabulary, role_vocabulary,
unk_word_id, unk_role_id, missing_word_id,
using_dropout, dropout_rate, optimizer,
loss, metrics)
# minus 1 here because one of the role is target role
self.input_length = n_role_vocab - 1
# each input is a fixed window of frame set, each word correspond to one role
input_words = Input(
shape=(self.input_length, ), dtype=tf.uint32,
name='input_words') # Switched dtype to tf specific (team1-change)
input_roles = Input(
shape=(self.input_length, ), dtype=tf.uint32,
name='input_roles') # Switched dtype to tf specific (team1-change)
target_role = Input(
shape=(1, ), dtype=tf.uint32,
name='target_role') # Switched dtype to tf specific (team1-change)
# role based embedding layer
embedding_layer = role_based_word_embedding(
input_words, input_roles, n_word_vocab, n_role_vocab,
glorot_uniform(), missing_word_id, self.input_length,
n_factors_emb, True, using_dropout, dropout_rate)
# sum on input_length direction;
# obtaining context embedding layer, shape is (batch_size, n_factors_emb)
event_embedding = Lambda(
lambda x: K.sum(x, axis=1),
name='event_embedding',
output_shape=(n_factors_emb, ))(embedding_layer)
# fully connected layer, output shape is (batch_size, input_length, n_hidden)
hidden = Dense(n_hidden,
activation='linear',
input_shape=(n_factors_emb, ),
name='projected_event_embedding')(event_embedding)
# non-linear layer, using 1 to initialize
non_linearity = PReLU(alpha_initializer='ones',
name='context_embedding')(hidden)
# hidden layer
hidden_layer2 = target_word_hidden(non_linearity,
target_role,
n_word_vocab,
n_role_vocab,
glorot_uniform(),
n_factors_cls,
n_hidden,
using_dropout=using_dropout,
dropout_rate=dropout_rate)
# softmax output layer
output_layer = Dense(n_word_vocab,
activation='softmax',
input_shape=(n_factors_cls, ),
name='softmax_word_output')(hidden_layer2)
self.model = Model(inputs=[input_words, input_roles, target_role],
outputs=[output_layer])
self.model.compile(optimizer, loss, metrics)
def set_0_bias(self):
""" This function is used as a hack that set output bias to 0.
According to Ottokar's advice in the paper, during the *evaluation*, the output bias needs to be 0
in order to replicate the best performance reported in the paper.
"""
word_output_weights = self.model.get_layer(
"softmax_word_output").get_weights()
word_output_kernel = word_output_weights[0]
word_output_bias = np.zeros(self.n_word_vocab)
self.model.get_layer("softmax_word_output").set_weights(
[word_output_kernel, word_output_bias])
return word_output_weights[1]
def set_bias(self, bias):
word_output_weights = self.model.get_layer(
"softmax_word_output").get_weights()
word_output_kernel = word_output_weights[0]
self.model.get_layer("softmax_word_output").set_weights(
[word_output_kernel, bias])
return bias
# Deprecated temporarily
def train(self,
i_w,
i_r,
t_w,
t_r,
t_w_c,
t_r_c,
batch_size=256,
epochs=100,
validation_split=0.05,
verbose=0):
train_result = self.model.fit([i_w, i_r, t_r], t_w_c, batch_size,
epochs, validation_split, verbose)
return train_result
def test(self,
i_w,
i_r,
t_w,
t_r,
t_w_c,
t_r_c,
batch_size=256,
verbose=0):
test_result = self.model.evaluate([i_w, i_r, t_r], t_w_c, batch_size,
verbose)
return test_result
def train_on_batch(self, i_w, i_r, t_w, t_r, t_w_c, t_r_c):
train_result = self.model.train_on_batch([i_w, i_r, t_r], t_w_c)
return train_result
def test_on_batch(self,
i_w,
i_r,
t_w,
t_r,
t_w_c,
t_r_c,
sample_weight=None):
test_result = self.model.test_on_batch([i_w, i_r, t_r], t_w_c,
sample_weight)
return test_result
def predict(self, i_w, i_r, t_r, batch_size=1, verbose=0):
""" Return the output from softmax layer. """
predict_result = self.model.predict([i_w, i_r, t_r], batch_size,
verbose)
return predict_result
def summary(self):
self.model.summary()
def predict_class(self, i_w, i_r, t_r, batch_size=1, verbose=0):
""" Return predicted target word from prediction. """
predict_result = self.predict(i_w, i_r, t_r, batch_size, verbose)
return np.argmax(predict_result, axis=1)
def p_words(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0):
""" Return the output scores given target words. """
predict_result = self.predict(i_w, i_r, t_r, batch_size, verbose)
return predict_result[range(batch_size), list(t_w)]
def top_words(self, i_w, i_r, t_r, topN=20, batch_size=1, verbose=0):
""" Return top N target words given context. """
predict_result = self.predict(i_w, i_r, t_r, batch_size, verbose)
rank_list = np.argsort(predict_result, axis=1)
return [r[-topN:][::-1] for r in rank_list]
def list_top_words(self, i_w, i_r, t_r, topN=20, batch_size=1, verbose=0):
""" Return a list of decoded top N target words.
(Only for reference, can be removed.)
"""
top_words_lists = self.top_words(i_w, i_r, t_r, topN, batch_size,
verbose)
print(
type(top_words_lists)) # Updated to python3 syntax (team1-change)
result = []
for i in range(batch_size):
top_words_list = top_words_lists[i]
result.append([self.word_decoder[w] for w in top_words_list])
return result
class SGAN:
def __init__(self, args):
self.input_shape = 28
self.num_classes = 2
self.latent_dim = 100
optimizer = Adam(0.0002, 0.5)
# Build and compile the discriminator
self.discriminator = self.build_discriminator()
self.discriminator.compile(
loss=['binary_crossentropy', 'categorical_crossentropy'],
loss_weights=[0.5, 0.5],
optimizer=optimizer,
metrics=['accuracy'])
# Build the generator
self.generator = self.build_generator()
# The generator takes noise as input and generates imgs
noise = Input(shape=(64, ))
img = self.generator(noise)
# For the combined model we will only train the generator
self.discriminator.trainable = False
# The valid takes generated images as input and determines validity
valid, _ = self.discriminator(img)
# The combined model (stacked generator and discriminator)
# Trains generator to fool discriminator
self.combined = Model(noise, valid)
self.combined.compile(loss=['binary_crossentropy'],
optimizer=optimizer)
def build_generator(self):
model = Sequential()
model.add(Dense(78, activation="relu", input_dim=self.latent_dim))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(56, activation="relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(32, activation="relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(28, activation="tanh"))
model.summary()
noise = Input(shape=(self.latent_dim, ))
img = model(noise)
return Model(noise, img)
def build_discriminator(self):
model = Sequential()
model.add(Dense(78, activation="relu", input_dim=self.input_shape))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(56, activation="relu"))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(32, activation="relu"))
model.add(Dropout(rate=0.3))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(28, activation="relu"))
model.add(Dropuout(rate=0.3))
model.add(BatchNormalization(momentum=0.8))
model.add(Dense(10, activation="relu"))
model.summary()
img = Input(shape=self.img_shape)
features = model(img)
valid = Dense(1, activation="sigmoid")(features)
label = Dense(self.num_classes + 1, activation="softmax")(features)
return Model(img, [valid, label])
def train(self, epochs, batch_size=100, sample_interval=50):
# Load the dataset
data = data_load("credit_fraud_sampled.csv")
x_train, y_train = data[:2]
x_val, y_val = data[2:4]
x_test, y_test = data[4:]
# Class weights:
# To balance the difference in occurences of digit class labels.
# 50% of labels that the discriminator trains on are 'fake'.
# Weight = 1 / frequency
half_batch = batch_size // 2
cw1 = {0: 1, 1: 1}
cw2 = {
i: self.num_classes / half_batch
for i in range(self.num_classes)
}
cw2[self.num_classes] = 1 / half_batch
# Adversarial ground truths
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
for epoch in range(epochs):
# ---------------------
# Train Discriminator
# ---------------------
# Select a random batch of images
idx = np.random.randint(0, X_train.shape[0], batch_size)
imgs = X_train[idx]
# Sample noise and generate a batch of new images
noise = np.random.normal(0, 1, (batch_size, self.latent_dim))
gen_imgs = self.generator.predict(noise)
# One-hot encoding of labels
labels = to_categorical(y_train[idx],
num_classes=self.num_classes + 1)
fake_labels = to_categorical(np.full((batch_size, 1),
self.num_classes),
num_classes=self.num_classes + 1)
# Train the discriminator
d_loss_real = self.discriminator.train_on_batch(
imgs, [valid, labels], class_weight=[cw1, cw2])
d_loss_fake = self.discriminator.train_on_batch(
gen_imgs, [fake, fake_labels], class_weight=[cw1, cw2])
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
# ---------------------
# Train Generator
# ---------------------
g_loss = self.combined.train_on_batch(noise,
valid,
class_weight=[cw1, cw2])
# Plot the progress
if epoch % 100 == 0:
print(
"%d [D loss: %f, acc: %.2f%%, op_acc: %.2f%%] [G loss: %f]"
% (epoch, d_loss[0], 100 * d_loss[3], 100 * d_loss[4],
g_loss))
# If at save interval => save generated image samples
if epoch % sample_interval == 0:
self.sample_images(epoch)
def sample_images(self, epoch):
r, c = 5, 5
noise = np.random.normal(0, 1, (r * c, self.latent_dim))
gen_imgs = self.generator.predict(noise)
# Rescale images 0 - 1
gen_imgs = 0.5 * gen_imgs + 0.5
fig, axs = plt.subplots(r, c)
cnt = 0
for i in range(r):
for j in range(c):
axs[i, j].imshow(gen_imgs[cnt, :, :, 0], cmap='gray')
axs[i, j].axis('off')
cnt += 1
fig.savefig("images/mnist_%d.png" % epoch)
plt.close()
def save_model(self):
def save(model, model_name):
model_path = "saved_model/%s.json" % model_name
weights_path = "saved_model/%s_weights.hdf5" % model_name
options = {"file_arch": model_path, "file_weight": weights_path}
json_string = model.to_json()
open(options['file_arch'], 'w').write(json_string)
model.save_weights(options['file_weight'])
save(self.generator, "mnist_sgan_generator")
save(self.discriminator, "mnist_sgan_discriminator")
save(self.combined, "mnist_sgan_adversarial")
class MTRFv4(GenericModel):
"""Multi-task non-incremental role-filler
"""
def __init__(self,
n_word_vocab=50001,
n_role_vocab=7,
n_factors_emb=300,
n_hidden=300,
word_vocabulary=None,
role_vocabulary=None,
unk_word_id=50000,
unk_role_id=7,
missing_word_id=50001,
using_dropout=False,
dropout_rate=0.3,
optimizer='adagrad',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'],
loss_weights=[1., 1.]):
super(MTRFv4, self).__init__(n_word_vocab, n_role_vocab, n_factors_emb,
n_hidden, word_vocabulary,
role_vocabulary, unk_word_id, unk_role_id,
missing_word_id, using_dropout,
dropout_rate, optimizer, loss, metrics)
# minus 1 here because one of the role is target role
input_length = n_role_vocab - 1
n_factors_cls = n_hidden
# each input is a fixed window of frame set, each word correspond to one role
input_words = Input(
shape=(input_length, ), dtype=tf.uint32,
name='input_words') # Switched dtype to tf specific (team1-change)
input_roles = Input(
shape=(input_length, ), dtype=tf.uint32,
name='input_roles') # Switched dtype to tf specific (team1-change)
target_word = Input(
shape=(1, ), dtype=tf.uint32,
name='target_word') # Switched dtype to tf specific (team1-change)
target_role = Input(
shape=(1, ), dtype=tf.uint32,
name='target_role') # Switched dtype to tf specific (team1-change)
# role based embedding layer
embedding_layer = factored_embedding(input_words, input_roles,
n_word_vocab, n_role_vocab,
glorot_uniform(), missing_word_id,
input_length, n_factors_emb,
n_hidden, True, using_dropout,
dropout_rate)
# non-linear layer, using 1 to initialize
non_linearity = PReLU(alpha_initializer='ones')(embedding_layer)
# mean on input_length direction;
# obtaining context embedding layer, shape is (batch_size, n_hidden)
context_embedding = Lambda(lambda x: K.mean(x, axis=1),
name='context_embedding',
output_shape=(n_hidden, ))(non_linearity)
# target word hidden layer
tw_hidden = target_word_hidden(context_embedding,
target_role,
n_word_vocab,
n_role_vocab,
glorot_uniform(),
n_hidden,
n_hidden,
using_dropout=using_dropout,
dropout_rate=dropout_rate)
# target role hidden layer
tr_hidden = target_role_hidden(context_embedding,
target_word,
n_word_vocab,
n_role_vocab,
glorot_uniform(),
n_hidden,
n_hidden,
using_dropout=using_dropout,
dropout_rate=dropout_rate)
# softmax output layer
target_word_output = Dense(n_word_vocab,
activation='softmax',
input_shape=(n_hidden, ),
name='softmax_word_output')(tw_hidden)
# softmax output layer
target_role_output = Dense(n_role_vocab,
activation='softmax',
input_shape=(n_hidden, ),
name='softmax_role_output')(tr_hidden)
self.model = Model(
inputs=[input_words, input_roles, target_word, target_role],
outputs=[target_word_output, target_role_output])
self.model.compile(optimizer, loss, metrics, loss_weights)
def set_0_bias(self):
word_output_weights = self.model.get_layer(
"softmax_word_output").get_weights()
word_output_kernel = word_output_weights[0]
word_output_bias = np.zeros(self.n_word_vocab)
self.model.get_layer("softmax_word_output").set_weights(
[word_output_kernel, word_output_bias])
role_output_weights = self.model.get_layer(
"softmax_role_output").get_weights()
role_output_kernel = role_output_weights[0]
role_output_bias = np.zeros(self.n_role_vocab)
self.model.get_layer("softmax_role_output").set_weights(
[role_output_kernel, role_output_bias])
return word_output_weights[1], role_output_weights[1]
def set_bias(self, bias):
word_output_weights = self.model.get_layer(
"softmax_word_output").get_weights()
word_output_kernel = word_output_weights[0]
self.model.get_layer("softmax_word_output").set_weights(
[word_output_kernel, bias[0]])
role_output_weights = self.model.get_layer(
"softmax_role_output").get_weights()
role_output_kernel = role_output_weights[0]
self.model.get_layer("softmax_role_output").set_weights(
[role_output_kernel, bias[1]])
return bias
# Train and test
# Deprecated temporarily
def train(self,
i_w,
i_r,
t_w,
t_r,
t_w_c,
t_r_c,
batch_size=256,
epochs=100,
validation_split=0.05,
verbose=0):
train_result = self.model.fit([i_w, i_r, t_w, t_r], [t_w_c, t_r_c],
batch_size, epochs, validation_split,
verbose)
return train_result
def test(self,
i_w,
i_r,
t_w,
t_r,
t_w_c,
t_r_c,
batch_size=256,
verbose=0):
test_result = self.model.evaluate([i_w, i_r, t_w, t_r], [t_w_c, t_r_c],
batch_size, verbose)
return test_result
def train_on_batch(self, i_w, i_r, t_w, t_r, t_w_c, t_r_c):
train_result = self.model.train_on_batch([i_w, i_r, t_w, t_r],
[t_w_c, t_r_c])
return train_result
def test_on_batch(self,
i_w,
i_r,
t_w,
t_r,
t_w_c,
t_r_c,
sample_weight=None):
test_result = self.model.test_on_batch([i_w, i_r, t_w, t_r],
[t_w_c, t_r_c], sample_weight)
return test_result
def predict(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0):
""" Return the output from softmax layer. """
predict_result = self.model.predict([i_w, i_r, t_w, t_r], batch_size,
verbose)
return predict_result
def predict_word(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0):
""" Return predicted target word from prediction. """
predict_result = self.predict(i_w, i_r, t_w, t_r, batch_size, verbose)
return np.argmax(predict_result[0], axis=1)
def predict_role(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0):
""" Return predicted target role from prediction. """
predict_result = self.predict(i_w, i_r, t_w, t_r, batch_size, verbose)
return np.argmax(predict_result[1], axis=1)
def p_words(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0):
""" Return the output scores given target words. """
predict_result = self.predict(i_w, i_r, t_w, t_r, batch_size, verbose)
return predict_result[0][range(batch_size), list(t_w)]
def p_roles(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0):
""" Return the output scores given target roles. """
predict_result = self.predict(i_w, i_r, t_w, t_r, batch_size, verbose)
return predict_result[1][range(batch_size), list(t_r)]
def top_words(self, i_w, i_r, t_w, t_r, topN=20, batch_size=1, verbose=0):
""" Return top N target words given context. """
predict_result = self.predict(i_w, i_r, t_w, t_r, batch_size,
verbose)[0]
rank_list = np.argsort(predict_result, axis=1)[0]
return rank_list[-topN:][::-1]
# return [r[-topN:][::-1] for r in rank_list]
# TODO
def list_top_words(self, i_w, i_r, t_r, topN=20, batch_size=1, verbose=0):
""" Return a list of decoded top N target words.
(Only for reference, can be removed.)
"""
top_words_lists = self.top_words(i_w, i_r, t_r, topN, batch_size,
verbose)
print(
type(top_words_lists)) # Updated to python3 syntax (team1-change)
result = []
for i in range(batch_size):
top_words_list = top_words_lists[i]
result.append([self.word_decoder[w] for w in top_words_list])
return result
def summary(self):
self.model.summary()