def eval_dom(self,
sess,
src_reviews,
tar_reviews,
src_labels,
tar_labels,
steps=None,
batch_size=None):
dom_preds = []
dom_labels = []
src_batch_generator = nn_utils.batch_generator(reviews=src_reviews,
labels=src_labels,
batch_size=batch_size,
shuffle=False)
tar_batch_generator = nn_utils.batch_generator(reviews=tar_reviews,
labels=tar_labels,
batch_size=batch_size,
shuffle=False)
for i in xrange(steps):
xb_s, yb_s = src_batch_generator.next_batch()
xb_t, yb_t = tar_batch_generator.next_batch()
xb_d = np.vstack((xb_s, xb_t))
dom_label = np.vstack((yb_s, yb_t))
dom_pred = self.predict_dom(sess, xb_d)
dom_preds += list(dom_pred)
dom_labels += list(dom_label)
dom_acc = metrics.accuracy_score(np.array(dom_preds),
np.argmax(dom_labels, axis=1))
return dom_acc
def eval_pivots(self,
sess,
reviews,
word_masks,
sent_masks,
u_labels,
v_labels,
batch_size=None):
u_preds = []
v_preds = []
batch_generator = nn_utils.batch_generator(reviews=reviews,
word_masks=word_masks,
sent_masks=sent_masks,
batch_size=batch_size,
shuffle=False)
for i in xrange(batch_generator.batch_num):
xb, wmb, smb = batch_generator.next_batch()
u_pred, v_pred = self.predict_pivots(sess, xb, wmb, smb)
u_preds += list(u_pred)
v_preds += list(v_pred)
u_acc = metrics.accuracy_score(np.array(u_preds),
np.argmax(u_labels, axis=1))
v_acc = metrics.accuracy_score(np.array(v_preds),
np.argmax(v_labels, axis=1))
return u_acc, v_acc
def evaluate_on_test(self, fit_model, x_test, y_test, n_classes, cfg):
fit_model.evaluate_generator(generator=nn_utils.batch_generator(x_test, y_test,
batch_size=self.batch_size,
shuffle=False,
one_hot_labels=True,
n_classes=n_classes),
samples=x_test.shape[0],
)
def supervised_classification(self, model, x_train, x_val, y_train, y_val, x_test, y_test, n_classes, final_act_fn='softmax', loss='categorical_crossentropy', get_recon_error=False):
model.add(Dense(n_classes, activation=final_act_fn))
model.compile(loss=loss, optimizer=self.optimizer)
# Early stopping to stop training when val loss increses for 1 epoch
early_stopping = EarlyStopping(
monitor='val_loss', patience=1, verbose=0)
hist = model.fit_generator(generator=nn_utils.batch_generator(x_train, y_train,
batch_size=self.batch_size,
shuffle=True,
n_classes=n_classes,
one_hot_labels=True),
samples_per_epoch=x_train.shape[0],
callbacks=[early_stopping],
nb_epoch=self.nb_epoch,
verbose=self.verbose,
validation_data=nn_utils.batch_generator(x_val, y_val,
batch_size=self.batch_size,
shuffle=False,
n_classes=n_classes,
one_hot_labels=True),
nb_val_samples=x_val.shape[0]
)
# get final hidden layer output of finetuned SDAE
final_train = self._get_intermediate_output(
model, x_train, n_layer=-2, train=0, n_out=model.layers[-2].output_shape[1], batch_size=self.batch_size)
final_val = self._get_intermediate_output(
model, x_val, n_layer=-2, train=0, n_out=model.layers[-2].output_shape[1], batch_size=self.batch_size)
if x_test is not None:
final_test = self._get_intermediate_output(
model, x_test, n_layer=-2, train=0, n_out=model.layers[-2].output_shape[1], batch_size=self.batch_size)
else:
final_test = None
# get reconstruction error of final nodes
if get_recon_error:
recon_mse = self._get_recon_error(model, x_train, n_out=n_classes)
else:
recon_mse = None
return model, (final_train, final_val, final_test), recon_mse
def predict(self, fit_model, x_test):
predictions = fit_model.predict_generator(generator=nn_utils.batch_generator(
x_test,
None,
batch_size=self.batch_size,
shuffle=False,
y_gen=False
),
val_samples=x_test.shape[0],
)
return predictions
def vis_attention(self, sess, reviews, batch_size=None):
p_w_atttns, p_s_attns = [], []
batch_generator = nn_utils.batch_generator(reviews,
batch_size=batch_size,
shuffle=False)
for i in xrange(batch_generator.batch_num):
xb = batch_generator.next_batch()
w1, s1 = self.get_attention(sess, xb)
p_w_atttns += list(w1)
p_s_attns += list(s1)
return np.array(p_w_atttns), np.array(p_s_attns)
def evaluate_on_test(self, fit_model, x_test, y_test, n_classes, cfg):
"""
Evaluate a trained model on test dataset
Use this function only for the final evaluation, not for development
"""
fit_model.evaluate_generator(
generator=nn_utils.batch_generator(x_test,
y_test,
batch_size=self.batch_size,
shuffle=False,
one_hot_labels=True,
n_classes=n_classes),
samples=x_test.shape[0],
)
def eval_sen(self, sess, reviews, sen_labels, batch_size=None):
preds = []
batch_generator = nn_utils.batch_generator(reviews=reviews,
batch_size=batch_size,
shuffle=False)
for i in xrange(batch_generator.batch_num):
xb = batch_generator.next_batch()
pred = self.predict_sen(sess, xb)
preds += list(pred)
preds = np.array(preds)
acc = metrics.accuracy_score(preds, np.argmax(sen_labels, axis=1))
return acc, preds
def predict(self, fit_model, x_test):
"""
Get prediction probability for each class for test data
@param fit_model: trained model
@param x_test: test data
"""
predictions = fit_model.predict_generator(
generator=nn_utils.batch_generator(x_test,
None,
batch_size=self.batch_size,
shuffle=False,
y_gen=False),
val_samples=x_test.shape[0],
)
return predictions
def vis_attention(self,
sess,
reviews,
word_masks,
sent_masks,
batch_size=None):
p_w_atttns, p_s_attns = [], []
np_w_atttns, np_s_attns = [], []
batch_generator = nn_utils.batch_generator(reviews=reviews,
word_masks=word_masks,
sent_masks=sent_masks,
batch_size=batch_size,
shuffle=False)
for i in xrange(batch_generator.batch_num):
xb, wmb, smb = batch_generator.next_batch()
w1, s1, w2, s2 = self.get_attention(sess, xb, wmb, smb)
p_w_atttns += list(w1)
p_s_attns += list(s1)
np_w_atttns += list(w2)
np_s_attns += list(s2)
return np.array(p_w_atttns), np.array(p_s_attns), np.array(
np_w_atttns), np.array(np_s_attns)
def get_pretrained_sda(self, data_in, data_val, data_test, dir_out, get_enc_model=True, write_model=True, model_layers=None):
if model_layers is not None:
self.n_layers = len(model_layers)
else:
model_layers = [None]*self.n_layers
encoders = []
recon_mse = 0
for cur_layer in range(self.n_layers):
if model_layers[cur_layer] is None:
input_layer = Input(shape=(data_in.shape[1],))
# masking input data to learn to generalize, and prevent identity learning
dropout_layer = Dropout(self.dropout[cur_layer])
in_dropout = dropout_layer(input_layer)
encoder_layer = Dense(output_dim=self.n_hid[cur_layer], init='glorot_uniform',
activation=self.enc_act[cur_layer], name='encoder'+str(cur_layer), bias=self.bias)
encoder = encoder_layer(in_dropout)
# same no. of output units as input units (to reconstruct the signal)
n_out = data_in.shape[1]
decoder_layer = Dense(output_dim=n_out, bias=self.bias, init='glorot_uniform',
activation=self.dec_act[cur_layer], name='decoder'+str(cur_layer))
decoder = decoder_layer(encoder)
cur_model = Model(input_layer, decoder)
cur_model.compile(loss=self.loss_fn, optimizer=self.optimizer)
# cur_model.summary()
else:
cur_model = model_layers[cur_layer]
print("Training layer " + str(cur_layer))
# Early stopping to stop training when val loss increases for 1 epoch
early_stopping = EarlyStopping(
monitor='val_loss', patience=1, verbose=0)
hist = cur_model.fit_generator(generator=nn_utils.batch_generator(
data_in, data_in,
batch_size=self.batch_size,
shuffle=True
),
callbacks=[early_stopping],
nb_epoch=self.nb_epoch,
samples_per_epoch=data_in.shape[0],
verbose=self.verbose,
validation_data=nn_utils.batch_generator(
data_val, data_val,
batch_size=self.batch_size,
shuffle=False),
nb_val_samples=data_val.shape[0]
)
print("Layer " + str(cur_layer) + " has been trained")
model_layers[cur_layer] = cur_model
encoder_layer = cur_model.layers[-2]
encoders.append(encoder_layer)
if cur_layer == 0:
recon_mse = self._get_recon_error(
cur_model, data_in, n_out=cur_model.layers[-1].output_shape[1])
# train = 0 because we do not want to use dropout to get hidden node value, since is a train-only behavior, used only to learn weights. output of second layer: hidden layer
data_in = self._get_intermediate_output(
cur_model, data_in, n_layer=2, train=0, n_out=self.n_hid[cur_layer], batch_size=self.batch_size)
assert data_in.shape[1] == self.n_hid[cur_layer], "Output of hidden layer not retrieved"
# get output of second layer (hidden layer) without dropout
data_val = self._get_intermediate_output(
cur_model, data_val, n_layer=2, train=0, n_out=self.n_hid[cur_layer], batch_size=self.batch_size)
data_test = self._get_intermediate_output(
cur_model, data_test, n_layer=2, train=0, n_out=self.n_hid[cur_layer], batch_size=self.batch_size)
self._write_sda_config(dir_out)
if get_enc_model:
final_model = self._build_model_from_encoders(
encoders, dropout_all=False) # , final_act_fn= final_act_fn)
if write_model:
nn_utils.save_model(final_model, out_dir=dir_out, f_arch='enc_layers.png',
f_model='enc_layers.json', f_weights='enc_layers_weights.h5')
else:
final_model = model_layers
return final_model, (data_in, data_val, data_test), recon_mse
def get_pretrained_sda(self,
data_in,
data_val,
data_test,
dir_out,
get_enc_model=True,
write_model=True,
model_layers=None):
'''
Pretrains layers of a stacked denoising autoencoder to generate low-dimensional representation of data.
Returns a supervised model with pretrained encoding layers and a softmax layer on top, or a list of pretrained sdae layers.
Additionally, returns dense representation of input, validation and test data.
This dense representation is the value of the hidden node of the last layer.
The cur_model be used in supervised task by adding a classification/regression layer on top, or the dense pretrained data can be used as input of another cur_model.
@param data_in: input data (scipy sparse matrix supported)
@param data_val: validation data (scipy sparse matrix supported)
@param data_test: test data (scipy sparse matrix supported)
@param dir_out: output directory to write cur_model
@param get_enc_model: True to get supervised cur_model from SDAE
@param write_model: True to write cur_model to file
@param model_layers: Pretrained cur_model layers, to continue training pretrained model_layers, if required
'''
if model_layers is not None:
self.n_layers = len(model_layers)
else:
model_layers = [None] * self.n_layers
encoders = []
recon_mse = 0
for cur_layer in range(self.n_layers):
if model_layers[cur_layer] is None:
input_layer = Input(shape=(data_in.shape[1], ))
# masking input data to learn to generalize, and prevent identity learning
dropout_layer = Dropout(self.dropout[cur_layer])
in_dropout = dropout_layer(input_layer)
encoder_layer = Dense(output_dim=self.n_hid[cur_layer],
init='glorot_uniform',
activation=self.enc_act[cur_layer],
name='encoder' + str(cur_layer),
bias=self.bias)
encoder = encoder_layer(in_dropout)
n_out = data_in.shape[
1] #same no. of output units as input units (to reconstruct the signal)
decoder_layer = Dense(output_dim=n_out,
bias=self.bias,
init='glorot_uniform',
activation=self.dec_act[cur_layer],
name='decoder' + str(cur_layer))
decoder = decoder_layer(encoder)
cur_model = Model(input_layer, decoder)
cur_model.compile(loss=self.loss_fn, optimizer=self.optimizer)
# cur_model.summary()
else:
cur_model = model_layers[cur_layer]
print("Training layer " + str(cur_layer))
#Early stopping to stop training when val loss increases for 1 epoch
early_stopping = EarlyStopping(monitor='val_loss',
patience=1,
verbose=0)
hist = cur_model.fit_generator(
generator=nn_utils.batch_generator(data_in,
data_in,
batch_size=self.batch_size,
shuffle=True),
callbacks=[early_stopping],
nb_epoch=self.nb_epoch,
samples_per_epoch=data_in.shape[0],
verbose=self.verbose,
validation_data=nn_utils.batch_generator(
data_val,
data_val,
batch_size=self.batch_size,
shuffle=False),
nb_val_samples=data_val.shape[0])
print("Layer " + str(cur_layer) + " has been trained")
model_layers[cur_layer] = cur_model
encoder_layer = cur_model.layers[-2]
encoders.append(encoder_layer)
if cur_layer == 0:
recon_mse = self._get_recon_error(
cur_model,
data_in,
n_out=cur_model.layers[-1].output_shape[1])
data_in = self._get_intermediate_output(
cur_model,
data_in,
n_layer=2,
train=0,
n_out=self.n_hid[cur_layer],
batch_size=self.batch_size
) #train = 0 because we do not want to use dropout to get hidden node value, since is a train-only behavior, used only to learn weights. output of second layer: hidden layer
assert data_in.shape[1] == self.n_hid[
cur_layer], "Output of hidden layer not retrieved"
data_val = self._get_intermediate_output(
cur_model,
data_val,
n_layer=2,
train=0,
n_out=self.n_hid[cur_layer],
batch_size=self.batch_size
) #get output of second layer (hidden layer) without dropout
data_test = self._get_intermediate_output(
cur_model,
data_test,
n_layer=2,
train=0,
n_out=self.n_hid[cur_layer],
batch_size=self.batch_size)
self._write_sda_config(dir_out)
if get_enc_model:
final_model = self._build_model_from_encoders(
encoders, dropout_all=False) #, final_act_fn= final_act_fn)
if write_model:
nn_utils.save_model(final_model,
out_dir=dir_out,
f_arch='enc_layers.png',
f_model='enc_layers.json',
f_weights='enc_layers_weights.h5')
else:
final_model = model_layers
return final_model, (data_in, data_val, data_test), recon_mse
def get_pretrained_sda(self,
data_in,
data_val,
data_test,
shuffle=False,
seed=1337):
'''
Pretrains layers of a stacked denoising autoencoder to generate low-dimensional representation of data.
Returns a model with pretrained encoding layers. Additionally, returns dense representation of input, validation and test data.
This dense representation is the value of the hidden node of the last layer.
The model be used in supervised task by adding a classification/regression layer on top, or the dense pretrained data can be used as input of another model.
@param data_in: input data (scipy sparse matrix supported)
@param data_val: validation data (scipy sparse matrix supported)
@param data_test: test data (scipy sparse matrix supported)
@param shuffle: True to shuffle data before training model
@param seed: seed for random shuffling
'''
encoders = []
for cur_layer in range(self.n_layers):
model = Sequential()
# masking input data to learn to generalize, and prevent identity learning
in_dropout = Dropout(self.dropout[cur_layer],
input_shape=(data_in.shape[1], ))
model.add(in_dropout)
encoder = Dense(output_dim=self.n_hid[cur_layer],
init='glorot_uniform',
activation=self.enc_act[cur_layer],
name='encoder' + str(cur_layer),
bias=self.bias)
model.add(encoder)
n_out = data_in.shape[
1] #same no. of output units as input units (to reconstruct the signal)
decoder = Dense(output_dim=n_out,
bias=self.bias,
init='glorot_uniform',
activation=self.dec_act[cur_layer],
name='decoder' + str(cur_layer))
model.add(decoder)
model.compile(loss=self.loss_fn, optimizer=self.optimizer)
print("Training layer " + str(cur_layer))
#Early stopping to stop training when val loss increses for 1 epoch
early_stopping = EarlyStopping(monitor='val_loss',
patience=1,
verbose=0)
model.fit_generator(generator=nn_utils.batch_generator(
data_in,
data_in,
batch_size=self.batch_size,
shuffle=shuffle,
seed=seed),
callbacks=[early_stopping],
nb_epoch=self.nb_epoch,
samples_per_epoch=data_in.shape[0],
verbose=self.verbose,
validation_data=nn_utils.batch_generator(
data_val,
data_val,
batch_size=self.batch_size,
shuffle=shuffle,
seed=seed),
nb_val_samples=data_val.shape[0])
print("Layer " + str(cur_layer) + " has been trained")
encoders.append(encoder)
data_in = self._get_intermediate_output(
model,
data_in,
n_layer=1,
train=0,
n_out=self.n_hid[cur_layer],
batch_size=self.batch_size,
shuffle=shuffle,
seed=seed
) #train = 0 because we do not want to use dropout to get hidden node value, since is a train-only behavior, used only to learn weights. output of first layer: hidden layer
# assert data_in.shape[1] == n_hid[cur_layer], "Output of hidden layer not retrieved"
data_val = self._get_intermediate_output(
model,
data_val,
n_layer=1,
train=0,
n_out=self.n_hid[cur_layer],
batch_size=self.batch_size,
shuffle=shuffle,
seed=seed
) #get output of first layer (hidden layer) without dropout
data_test = self._get_intermediate_output(
model,
data_test,
n_layer=1,
train=0,
n_out=self.n_hid[cur_layer],
batch_size=self.batch_size,
shuffle=shuffle,
seed=seed)
pretrained_model = self._build_model_from_encoders(encoders)
return pretrained_model, (data_in, data_val, data_test)