def train():
(X_test, y_test, y_conf) = load.load_test_data()
Y_test = np_utils.to_categorical(y_test, classes)
print(X_test.shape[0], 'test samples')
X_test = X_test.astype("float32")
X_test /= 255
datagen = ImageDataGenerator(rotation_range=30, width_shift_range=0.01, height_shift_range=0.01, horizontal_flip=True, vertical_flip=True)
t0=time.time()
for e in range(nb_epoch):
print ("******** Epoch %d ********" % (e+1))
print ("Epoch Number: " + str(e))
for X_batch, y_batch, class_weight in BatchGenerator():
datagen.fit(X_batch)
model.fit_generator(datagen.flow(X_batch, y_batch, batch_size=18, shuffle=True),
callbacks=[lh,checkpointer],
samples_per_epoch=split_size,
nb_epoch=nb_epoch_per,
validation_data=(X_test,Y_test)
,class_weight=class_weight
)
y_pred = model.predict_classes(X_test, batch_size=20)
(accuracy, correct)=PredictionMatrix()
#model.save_weights((direct + '/weights/' + save_name[:-5] + 'E-%d.hdf5' ) % (e+1), overwrite=True)
#print ("Weights saved to " + direct + '/weights/' + save_name[:-5] + 'E-%d.hdf5' % (e+1))
t1=time.time()
tyme = t1-t0
print("Training completed in %f seconds" % tyme)
if save_name != '':
model.save_weights(direct + '/weights/' + save_name, overwrite=True)
print ("Weights saved to " + save_name)
print ("Final training weights saved to " + save_name)
return tyme
def testAll(args):
word2idx, vectors = create_model(args)
print("> Loading trained model and Test")
max_recall = load_model(args.model_dump)
print(f"max_recall: {max_recall}")
test_data = load_test_data(args, word2idx)
with torch.no_grad():
model.eval()
do_predict(args, test_data)
def testAll(args):
word2idx, vectors = create_model(args)
global idx2word
idx2word = {b:a for a,b in word2idx.items()}
print("> Loading trained model and Test")
max_recall = load_model(args.model_dump)
print(f"max_recall: {max_recall}")
test_data = load_test_data(args, word2idx)
with torch.no_grad():
model.eval()
do_predict(args, test_data, idx2word)
def load_test_rows(transform_dates=True,
transform_categorical_features=False,
add_timeseries_features=True):
"""
"""
data_rows = load.load_test_data()
historical_data = load.load_historical_test_data()
features = load.load_features()
timeseries_features = load.TIMESERIES_FEATURES
data_rows, features = transformations(add_timeseries_features, data_rows,
features, historical_data,
timeseries_features,
transform_categorical_features,
transform_dates)
return data_rows, features
def test(name, words, files):
class_count = len(files)
x, y, y_ = th.setup((len(words)), class_count)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(sess, name)
print("Model restored")
lines, inputs = load.load_test_data(words)
for i in range(len(lines)):
line = lines[i]
inp = inputs[i]
result = sess.run(y, feed_dict={x: inp})
m = 0.0
class_index = 0
for i in range(len(result[0])):
r = result[0][i]
if r > m:
m = r
class_index = i
print("Test data " + line + " : " + files[class_index] + " : " + str(result) + " : ")
def train():
(X_test, y_test) = load.load_test_data()
Y_test = np_utils.to_categorical(y_test, 2)
print(X_test.shape[0], 'test samples')
X_test = X_test.astype("float32")
X_test /= 255
datagen = ImageDataGenerator(fill_mode='constant', cval=0, rotation_range=10, width_shift_range=0.01, height_shift_range=0.01, horizontal_flip=True, vertical_flip=True)
if split_size > train_size:
print('Training...')
t0 = time.time()
(X_train, Y_train, class_weight) = load.load_train_data(split=0, split_size=train_size)
Y_train = np_utils.to_categorical(y_train, classes)
print(X_train.shape[0], 'test samples')
X_train = X_train.astype("float32")
X_train /= 255
datagen.fit(X_train)
for e in range(nb_epoch):
print ("******** Epoch %d ********" % (e+1))
model.fit_generator(datagen.flow(X_train, Y_train, shuffle=True, batch_size=15),
callbacks=[lh,checkpointer],
samples_per_epoch=split_size,
nb_epoch=nb_epoch_per,
validation_data=(X_test,Y_test)
,class_weight=class_weight
)
model.save_weights((direct + '/weights/' + save_name[:-5] + 'E-%d.hdf5' ) % (e+1), overwrite=True)
print ("Weights saved to " + direct + '/weights/' + save_name[:-5] + 'E-%d.hdf5' % (e+1))
elif split_size < train_size:
print ("Using batch training...")
t0=time.time()
for e in range(nb_epoch):
print ("******** Epoch %d ********" % (e+1))
print ("Epoch Number: " + str(e))
for X_batch, y_batch, class_weight, split in BatchGenerator():
datagen.fit(X_batch)
#if split > train_size/split_size:
#print ("Error in splits")
#break
model.fit_generator(datagen.flow(X_batch, y_batch, batch_size=15, shuffle=True
#,save_to_dir='/media/harry/1TB/BinaryNew/da/'
),
callbacks=[lh,checkpointer],
samples_per_epoch=split_size,
nb_epoch=nb_epoch_per,
validation_data=(X_test,Y_test)
,class_weight=class_weight
)
#test_data = X_test[:101]
#test_labels = model.predict_classes(test_data)
#print (test_labels)
model.save_weights((direct + '/weights/' + save_name[:-5] + 'E-%d-%d.hdf5' ) % (e+1, split), overwrite=True)
print ("Weights saved to " + direct + '/weights/' + save_name[:-5] + 'E-%d-%d.hdf5' % (e+1, split))
t1=time.time()
tyme = t1-t0
print("Training completed in %f seconds" % tyme)
if save_name != '':
model.save_weights(direct + '/weights/' + save_name, overwrite=True)
print ("Weights saved to " + save_name)
print ("Final training weights saved to " + save_name)
return tyme
#!/usr/bin/env python3
import numpy as np
import torch
import sklearn
import sklearn.model_selection
from sklearn.preprocessing import FunctionTransformer
from sklearn.pipeline import Pipeline
import preprocess.glove_vectorize as glove_vectorize
import models
import load
X_train_df, y_train_df = load.load_train_data()
y = torch.from_numpy(y_train_df.to_numpy()).type(dtype=torch.FloatTensor)
X_test_df = load.load_test_data()
glove_vectorizer = glove_vectorize.get_transformer_train()
torch_converter = FunctionTransformer(torch.from_numpy)
type_caster = FunctionTransformer(lambda t: t.type(dtype=torch.FloatTensor))
sentence_length, vector_size = glove_vectorize.get_instance_dims()
cnn_model = models.CNNModel(vector_size=vector_size,
sentence_length=sentence_length)
clf = cnn_model.get_sklearn_compatible_estimator()
glove_CNN_pipeline = Pipeline([
('glove_vectorizer', glove_vectorizer),
('torch_converter', torch_converter),
('type_caster', type_caster),
('cnn_classifier', clf),
(j, self.evaluate_test(test_x, test_y), len(test_x)))
def evaluate_test(self, test_data_x, test_data_y):
result = [(np.argmax(self.feed_forward(x)), y)
for x, y in zip(test_data_x, test_data_y)]
return sum((int(x == y) for x, y in result))
def show_img(self, x, y):
print(y)
x = np.reshape(x, (28, 28))
p = plt.imshow(x, shape=(28, 28))
plt.show(p)
if __name__ == '__main__':
start_time = timeit.default_timer()
x, y = load.load_training_data()
test_x, test_y = load.load_test_data()
net = NeuralNetwork([784, 30, 10])
print('before training')
print(net.feed_forward(x[0]))
print("epch=before training %r/%r" %
(net.evaluate_test(test_x, test_y), len(test_x)))
net.update_weights(x, y, 3, 30, test_x, test_y, 10)
print('after training')
print(net.feed_forward(x[0]))
print(y[0])
stop_time = timeit.default_timer()
progress_time = stop_time - start_time
print('Time=', progress_time)