class NNRF(GenericModel): """Non-incremental model role-filler """ def __init__(self, n_word_vocab=50001, n_role_vocab=7, n_factors_emb=256, n_factors_cls=512, n_hidden=256, word_vocabulary={}, role_vocabulary={}, unk_word_id=50000, unk_role_id=7, missing_word_id=50001, using_dropout=False, dropout_rate=0.3, optimizer='adagrad', loss='sparse_categorical_crossentropy', metrics=['accuracy']): super(NNRF, self).__init__(n_word_vocab, n_role_vocab, n_factors_emb, n_hidden, word_vocabulary, role_vocabulary, unk_word_id, unk_role_id, missing_word_id, using_dropout, dropout_rate, optimizer, loss, metrics) # minus 1 here because one of the role is target role self.input_length = n_role_vocab - 1 # each input is a fixed window of frame set, each word correspond to one role input_words = Input( shape=(self.input_length, ), dtype=tf.uint32, name='input_words') # Switched dtype to tf specific (team1-change) input_roles = Input( shape=(self.input_length, ), dtype=tf.uint32, name='input_roles') # Switched dtype to tf specific (team1-change) target_role = Input( shape=(1, ), dtype=tf.uint32, name='target_role') # Switched dtype to tf specific (team1-change) # role based embedding layer embedding_layer = role_based_word_embedding( input_words, input_roles, n_word_vocab, n_role_vocab, glorot_uniform(), missing_word_id, self.input_length, n_factors_emb, True, using_dropout, dropout_rate) # sum on input_length direction; # obtaining context embedding layer, shape is (batch_size, n_factors_emb) event_embedding = Lambda( lambda x: K.sum(x, axis=1), name='event_embedding', output_shape=(n_factors_emb, ))(embedding_layer) # fully connected layer, output shape is (batch_size, input_length, n_hidden) hidden = Dense(n_hidden, activation='linear', input_shape=(n_factors_emb, ), name='projected_event_embedding')(event_embedding) # non-linear layer, using 1 to initialize non_linearity = PReLU(alpha_initializer='ones', name='context_embedding')(hidden) # hidden layer hidden_layer2 = target_word_hidden(non_linearity, target_role, n_word_vocab, n_role_vocab, glorot_uniform(), n_factors_cls, n_hidden, using_dropout=using_dropout, dropout_rate=dropout_rate) # softmax output layer output_layer = Dense(n_word_vocab, activation='softmax', input_shape=(n_factors_cls, ), name='softmax_word_output')(hidden_layer2) self.model = Model(inputs=[input_words, input_roles, target_role], outputs=[output_layer]) self.model.compile(optimizer, loss, metrics) def set_0_bias(self): """ This function is used as a hack that set output bias to 0. According to Ottokar's advice in the paper, during the *evaluation*, the output bias needs to be 0 in order to replicate the best performance reported in the paper. """ word_output_weights = self.model.get_layer( "softmax_word_output").get_weights() word_output_kernel = word_output_weights[0] word_output_bias = np.zeros(self.n_word_vocab) self.model.get_layer("softmax_word_output").set_weights( [word_output_kernel, word_output_bias]) return word_output_weights[1] def set_bias(self, bias): word_output_weights = self.model.get_layer( "softmax_word_output").get_weights() word_output_kernel = word_output_weights[0] self.model.get_layer("softmax_word_output").set_weights( [word_output_kernel, bias]) return bias # Deprecated temporarily def train(self, i_w, i_r, t_w, t_r, t_w_c, t_r_c, batch_size=256, epochs=100, validation_split=0.05, verbose=0): train_result = self.model.fit([i_w, i_r, t_r], t_w_c, batch_size, epochs, validation_split, verbose) return train_result def test(self, i_w, i_r, t_w, t_r, t_w_c, t_r_c, batch_size=256, verbose=0): test_result = self.model.evaluate([i_w, i_r, t_r], t_w_c, batch_size, verbose) return test_result def train_on_batch(self, i_w, i_r, t_w, t_r, t_w_c, t_r_c): train_result = self.model.train_on_batch([i_w, i_r, t_r], t_w_c) return train_result def test_on_batch(self, i_w, i_r, t_w, t_r, t_w_c, t_r_c, sample_weight=None): test_result = self.model.test_on_batch([i_w, i_r, t_r], t_w_c, sample_weight) return test_result def predict(self, i_w, i_r, t_r, batch_size=1, verbose=0): """ Return the output from softmax layer. """ predict_result = self.model.predict([i_w, i_r, t_r], batch_size, verbose) return predict_result def summary(self): self.model.summary() def predict_class(self, i_w, i_r, t_r, batch_size=1, verbose=0): """ Return predicted target word from prediction. """ predict_result = self.predict(i_w, i_r, t_r, batch_size, verbose) return np.argmax(predict_result, axis=1) def p_words(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0): """ Return the output scores given target words. """ predict_result = self.predict(i_w, i_r, t_r, batch_size, verbose) return predict_result[range(batch_size), list(t_w)] def top_words(self, i_w, i_r, t_r, topN=20, batch_size=1, verbose=0): """ Return top N target words given context. """ predict_result = self.predict(i_w, i_r, t_r, batch_size, verbose) rank_list = np.argsort(predict_result, axis=1) return [r[-topN:][::-1] for r in rank_list] def list_top_words(self, i_w, i_r, t_r, topN=20, batch_size=1, verbose=0): """ Return a list of decoded top N target words. (Only for reference, can be removed.) """ top_words_lists = self.top_words(i_w, i_r, t_r, topN, batch_size, verbose) print( type(top_words_lists)) # Updated to python3 syntax (team1-change) result = [] for i in range(batch_size): top_words_list = top_words_lists[i] result.append([self.word_decoder[w] for w in top_words_list]) return result
class SGAN: def __init__(self, args): self.input_shape = 28 self.num_classes = 2 self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile( loss=['binary_crossentropy', 'categorical_crossentropy'], loss_weights=[0.5, 0.5], optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generates imgs noise = Input(shape=(64, )) img = self.generator(noise) # For the combined model we will only train the generator self.discriminator.trainable = False # The valid takes generated images as input and determines validity valid, _ = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains generator to fool discriminator self.combined = Model(noise, valid) self.combined.compile(loss=['binary_crossentropy'], optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(78, activation="relu", input_dim=self.latent_dim)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(56, activation="relu")) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(32, activation="relu")) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(28, activation="tanh")) model.summary() noise = Input(shape=(self.latent_dim, )) img = model(noise) return Model(noise, img) def build_discriminator(self): model = Sequential() model.add(Dense(78, activation="relu", input_dim=self.input_shape)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(56, activation="relu")) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(32, activation="relu")) model.add(Dropout(rate=0.3)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(28, activation="relu")) model.add(Dropuout(rate=0.3)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(10, activation="relu")) model.summary() img = Input(shape=self.img_shape) features = model(img) valid = Dense(1, activation="sigmoid")(features) label = Dense(self.num_classes + 1, activation="softmax")(features) return Model(img, [valid, label]) def train(self, epochs, batch_size=100, sample_interval=50): # Load the dataset data = data_load("credit_fraud_sampled.csv") x_train, y_train = data[:2] x_val, y_val = data[2:4] x_test, y_test = data[4:] # Class weights: # To balance the difference in occurences of digit class labels. # 50% of labels that the discriminator trains on are 'fake'. # Weight = 1 / frequency half_batch = batch_size // 2 cw1 = {0: 1, 1: 1} cw2 = { i: self.num_classes / half_batch for i in range(self.num_classes) } cw2[self.num_classes] = 1 / half_batch # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # Sample noise and generate a batch of new images noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) gen_imgs = self.generator.predict(noise) # One-hot encoding of labels labels = to_categorical(y_train[idx], num_classes=self.num_classes + 1) fake_labels = to_categorical(np.full((batch_size, 1), self.num_classes), num_classes=self.num_classes + 1) # Train the discriminator d_loss_real = self.discriminator.train_on_batch( imgs, [valid, labels], class_weight=[cw1, cw2]) d_loss_fake = self.discriminator.train_on_batch( gen_imgs, [fake, fake_labels], class_weight=[cw1, cw2]) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- g_loss = self.combined.train_on_batch(noise, valid, class_weight=[cw1, cw2]) # Plot the progress if epoch % 100 == 0: print( "%d [D loss: %f, acc: %.2f%%, op_acc: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100 * d_loss[3], 100 * d_loss[4], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i, j].imshow(gen_imgs[cnt, :, :, 0], cmap='gray') axs[i, j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() def save_model(self): def save(model, model_name): model_path = "saved_model/%s.json" % model_name weights_path = "saved_model/%s_weights.hdf5" % model_name options = {"file_arch": model_path, "file_weight": weights_path} json_string = model.to_json() open(options['file_arch'], 'w').write(json_string) model.save_weights(options['file_weight']) save(self.generator, "mnist_sgan_generator") save(self.discriminator, "mnist_sgan_discriminator") save(self.combined, "mnist_sgan_adversarial")
class MTRFv4(GenericModel): """Multi-task non-incremental role-filler """ def __init__(self, n_word_vocab=50001, n_role_vocab=7, n_factors_emb=300, n_hidden=300, word_vocabulary=None, role_vocabulary=None, unk_word_id=50000, unk_role_id=7, missing_word_id=50001, using_dropout=False, dropout_rate=0.3, optimizer='adagrad', loss='sparse_categorical_crossentropy', metrics=['accuracy'], loss_weights=[1., 1.]): super(MTRFv4, self).__init__(n_word_vocab, n_role_vocab, n_factors_emb, n_hidden, word_vocabulary, role_vocabulary, unk_word_id, unk_role_id, missing_word_id, using_dropout, dropout_rate, optimizer, loss, metrics) # minus 1 here because one of the role is target role input_length = n_role_vocab - 1 n_factors_cls = n_hidden # each input is a fixed window of frame set, each word correspond to one role input_words = Input( shape=(input_length, ), dtype=tf.uint32, name='input_words') # Switched dtype to tf specific (team1-change) input_roles = Input( shape=(input_length, ), dtype=tf.uint32, name='input_roles') # Switched dtype to tf specific (team1-change) target_word = Input( shape=(1, ), dtype=tf.uint32, name='target_word') # Switched dtype to tf specific (team1-change) target_role = Input( shape=(1, ), dtype=tf.uint32, name='target_role') # Switched dtype to tf specific (team1-change) # role based embedding layer embedding_layer = factored_embedding(input_words, input_roles, n_word_vocab, n_role_vocab, glorot_uniform(), missing_word_id, input_length, n_factors_emb, n_hidden, True, using_dropout, dropout_rate) # non-linear layer, using 1 to initialize non_linearity = PReLU(alpha_initializer='ones')(embedding_layer) # mean on input_length direction; # obtaining context embedding layer, shape is (batch_size, n_hidden) context_embedding = Lambda(lambda x: K.mean(x, axis=1), name='context_embedding', output_shape=(n_hidden, ))(non_linearity) # target word hidden layer tw_hidden = target_word_hidden(context_embedding, target_role, n_word_vocab, n_role_vocab, glorot_uniform(), n_hidden, n_hidden, using_dropout=using_dropout, dropout_rate=dropout_rate) # target role hidden layer tr_hidden = target_role_hidden(context_embedding, target_word, n_word_vocab, n_role_vocab, glorot_uniform(), n_hidden, n_hidden, using_dropout=using_dropout, dropout_rate=dropout_rate) # softmax output layer target_word_output = Dense(n_word_vocab, activation='softmax', input_shape=(n_hidden, ), name='softmax_word_output')(tw_hidden) # softmax output layer target_role_output = Dense(n_role_vocab, activation='softmax', input_shape=(n_hidden, ), name='softmax_role_output')(tr_hidden) self.model = Model( inputs=[input_words, input_roles, target_word, target_role], outputs=[target_word_output, target_role_output]) self.model.compile(optimizer, loss, metrics, loss_weights) def set_0_bias(self): word_output_weights = self.model.get_layer( "softmax_word_output").get_weights() word_output_kernel = word_output_weights[0] word_output_bias = np.zeros(self.n_word_vocab) self.model.get_layer("softmax_word_output").set_weights( [word_output_kernel, word_output_bias]) role_output_weights = self.model.get_layer( "softmax_role_output").get_weights() role_output_kernel = role_output_weights[0] role_output_bias = np.zeros(self.n_role_vocab) self.model.get_layer("softmax_role_output").set_weights( [role_output_kernel, role_output_bias]) return word_output_weights[1], role_output_weights[1] def set_bias(self, bias): word_output_weights = self.model.get_layer( "softmax_word_output").get_weights() word_output_kernel = word_output_weights[0] self.model.get_layer("softmax_word_output").set_weights( [word_output_kernel, bias[0]]) role_output_weights = self.model.get_layer( "softmax_role_output").get_weights() role_output_kernel = role_output_weights[0] self.model.get_layer("softmax_role_output").set_weights( [role_output_kernel, bias[1]]) return bias # Train and test # Deprecated temporarily def train(self, i_w, i_r, t_w, t_r, t_w_c, t_r_c, batch_size=256, epochs=100, validation_split=0.05, verbose=0): train_result = self.model.fit([i_w, i_r, t_w, t_r], [t_w_c, t_r_c], batch_size, epochs, validation_split, verbose) return train_result def test(self, i_w, i_r, t_w, t_r, t_w_c, t_r_c, batch_size=256, verbose=0): test_result = self.model.evaluate([i_w, i_r, t_w, t_r], [t_w_c, t_r_c], batch_size, verbose) return test_result def train_on_batch(self, i_w, i_r, t_w, t_r, t_w_c, t_r_c): train_result = self.model.train_on_batch([i_w, i_r, t_w, t_r], [t_w_c, t_r_c]) return train_result def test_on_batch(self, i_w, i_r, t_w, t_r, t_w_c, t_r_c, sample_weight=None): test_result = self.model.test_on_batch([i_w, i_r, t_w, t_r], [t_w_c, t_r_c], sample_weight) return test_result def predict(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0): """ Return the output from softmax layer. """ predict_result = self.model.predict([i_w, i_r, t_w, t_r], batch_size, verbose) return predict_result def predict_word(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0): """ Return predicted target word from prediction. """ predict_result = self.predict(i_w, i_r, t_w, t_r, batch_size, verbose) return np.argmax(predict_result[0], axis=1) def predict_role(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0): """ Return predicted target role from prediction. """ predict_result = self.predict(i_w, i_r, t_w, t_r, batch_size, verbose) return np.argmax(predict_result[1], axis=1) def p_words(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0): """ Return the output scores given target words. """ predict_result = self.predict(i_w, i_r, t_w, t_r, batch_size, verbose) return predict_result[0][range(batch_size), list(t_w)] def p_roles(self, i_w, i_r, t_w, t_r, batch_size=1, verbose=0): """ Return the output scores given target roles. """ predict_result = self.predict(i_w, i_r, t_w, t_r, batch_size, verbose) return predict_result[1][range(batch_size), list(t_r)] def top_words(self, i_w, i_r, t_w, t_r, topN=20, batch_size=1, verbose=0): """ Return top N target words given context. """ predict_result = self.predict(i_w, i_r, t_w, t_r, batch_size, verbose)[0] rank_list = np.argsort(predict_result, axis=1)[0] return rank_list[-topN:][::-1] # return [r[-topN:][::-1] for r in rank_list] # TODO def list_top_words(self, i_w, i_r, t_r, topN=20, batch_size=1, verbose=0): """ Return a list of decoded top N target words. (Only for reference, can be removed.) """ top_words_lists = self.top_words(i_w, i_r, t_r, topN, batch_size, verbose) print( type(top_words_lists)) # Updated to python3 syntax (team1-change) result = [] for i in range(batch_size): top_words_list = top_words_lists[i] result.append([self.word_decoder[w] for w in top_words_list]) return result def summary(self): self.model.summary()