def test_wvmodel_loaded_correctly(self):
"""Test for training model for sentense classification with Keras"""
import data_helpers
test_cnn = Test_test_cnn_classifier
# Prepare word2vec model
wvmodel = data_helpers.load_google_model()
# load data
x_all, y_all = test_cnn.createTrainInput(wvmodel, 300)
xv_all, yv_all = test_cnn.createVerificationInput(wvmodel, 300)
# delete model
del wvmodel
# calc reasonable elements in train set
notNullCount = 0
flattenx = x_all.flatten()
for x in flattenx:
notNullCount += 1
# check train input is not empty
self.assertNotEqual(notNullCount, 0)
# calc reasonable elements in verification set
flattenxv = xv_all.flatten()
notNullCount = 0
for x in flattenx:
notNullCount += 1
# check verification input is not empty
self.assertNotEqual(notNullCount, 0)
def test_conv_layer_merged_train(self):
"""Test for training model for sentense classification with Keras"""
import data_helpers
# Prepare word2vec model
wvmodel = data_helpers.load_google_model()
model = Test_test_cnn_classifier.createMergedConv2DLayer(
data_helpers.max_sent_len, data_helpers.wordvec_len,
[2, 3, 4, 5, 6])
x, y = Test_test_cnn_classifier.createTrainInput(wvmodel)
model.fit(x,
y,
batch_size=32,
nb_epoch=80,
validation_split=0.1,
verbose=2)
json_string = model.to_json()
filePath = os.path.dirname(os.path.abspath(__file__))
open(os.path.join(filePath, "Data", 'my_model_architecture.json'),
'w').write(json_string)
model.save_weights(
os.path.join(filePath, "Data", 'my_model_weights.h5'))
output = model.predict(x)
for i in range(len(y)):
print(str(output[i]) + " " + str(y[i]))
def test_conv_layer_merged_train_mult(self):
"""Test for training model for sentense classification with Keras"""
import data_helpers
test_cnn = Test_test_cnn_classifier
# Prepare word2vec model
wvmodel = data_helpers.load_google_model()
x_all, y_all = test_cnn.createTrainInput(wvmodel, 300)
xv_all, yv_all = test_cnn.createVerificationInput(wvmodel, 300)
del wvmodel
filePath = os.path.dirname(os.path.abspath(__file__))
ptnnPath = os.path.join(filePath, "Data", "PreTrainedModels")
# Define data params for training
lengths = [(i + 1) * 20 + 160 for i in range(5)]
nEpochs = [(i + 1) * 20 + 80 for i in range(2)]
n_grams_set = [[2, 3, 4, 5], [2, 3, 4, 5, 6]] #, [2,3,4,5,6,7]]
# Do training in cycle
for n_grams in n_grams_set:
for nEpoch in nEpochs:
for vecLen in lengths:
# Create training and verification sets
x = data_helpers.word2vec_dataset_to_len(x_all, vecLen)
y = y_all
xv = data_helpers.word2vec_dataset_to_len(xv_all, vecLen)
yv = yv_all
# Create NN
model = test_cnn.createMergedConv2DLayer(
data_helpers.max_sent_len, vecLen, n_grams)
# Train NN
model.fit(x,
y,
batch_size=32,
nb_epoch=nEpoch,
validation_split=0.1,
verbose=2)
# Convert model structure into json
json_string = model.to_json()
# Save model structure
netName = test_cnn.netName(n_grams, vecLen, nEpoch)
open(os.path.join(ptnnPath, netName + ".json"),
'w').write(json_string)
# Save model weights
model.save_weights(os.path.join(ptnnPath, netName + ".h5"))
def test_lstm_train_mult(self):
"""Test for training lstm model for sentense classification with Keras"""
import data_helpers
# Prepare word2vec model
wvmodel = data_helpers.load_google_model()
x_all, y_all = Test_lstm.createTrainInput(wvmodel, 300)
xv_all, yv_all = Test_lstm.createVerificationInput(wvmodel, 300)
del wvmodel
filePath = os.path.dirname(os.path.abspath(__file__))
ptnnPath = os.path.join(filePath, "Data", "PreTrainedModels")
# Define data params for training
lengths = [160, 200]
nEpochs = [i * 100 + 400 for i in range(6)]
numHidden = [50, 100]
# Do training in cycle
for nEpoch in nEpochs:
for vecLen in lengths:
for nh in numHidden:
# Create training and verification sets
x = data_helpers.word2vec_dataset_to_len2D(x_all, vecLen)
y = y_all
xv = data_helpers.word2vec_dataset_to_len2D(xv_all, vecLen)
yv = yv_all
# Create NN
model = Test_lstm.createLSTMNetwork(
data_helpers.max_sent_len, vecLen, nh)
# Train NN
model.fit(x,
y,
batch_size=32,
nb_epoch=nEpoch,
validation_split=0.1,
verbose=2)
# Convert model structure into json
json_string = model.to_json()
# Save model structure
netName = Test_lstm.netName(vecLen, nEpoch, nh)
open(os.path.join(ptnnPath, netName + ".json"),
'w').write(json_string)
# Save model weights
model.save_weights(os.path.join(ptnnPath, netName + ".h5"))
def test_load_and_verify_conv_net_50(self):
import data_helpers
# Prepare word2vec model
wvmodel = data_helpers.load_google_model()
preTrainedNets = [
"CNN_2_3_4_5_6_50_80_Carpeting_gpu",
"CNN_2_3_4_5_6_50_100_Carpeting_gpu",
"CNN_2_3_4_5_6_50_80_Carpeting_cpu"
]
x, y = Test_test_cnn_classifier.createVerificationInput(wvmodel, 50)
Test_test_cnn_classifier.RunSequenceOfPreTrainedNetworksOnSameInput(
preTrainedNets, x, y)
def setUpClass(cls):
cls.wvmodel = data_helpers.load_google_model()