def train(a):
global f
features, labels = dl.load_training_data()
features = preprocess(features)
correct = 0
fail = 0
iteration = 0
while(True):
writeLog(["ITERATION: #" , str(iteration) , "\n"])
print("iteration; ", iteration)
for i in range(len(labels)):
#if i % 500 == 0:
# print(i)
a.propagate(features[i], labels[i])
a.backpropagate(features[i], generate_example_output(labels[i]))
print(a.correct)
writeLog(["TRAINING: " , getPercentage(a.correct, a.fail) , "\n"])
print("TRAINING: " , getPercentage(a.correct, a.fail) , "\n")
res = test(a)
print("TESTING: ",res)
writeLog(["TESTING: " , res , "\n"])
iteration += 1
if (iteration > 15):
break
else:
a.correct = 0
a.fail = 0
def train(current_epoch, global_step, x=1):
model = Model(img_channels=cfg.img_channels, num_label=cfg.num_labels)
tf.logging.info(' Graph loaded')
with tf.Session(graph=model.graph,
config=tf.ConfigProto(allow_soft_placement=True)) as sess:
saver = tf.train.Saver(max_to_keep=2, save_relative_paths=True)
if str(tf.train.latest_checkpoint(cfg.logdir)) == 'None':
init = tf.initialize_all_variables()
sess.run(init)
print('No model weights to restore')
tf.logging.info('No model to restore!')
else:
saver.restore(sess, tf.train.latest_checkpoint(cfg.logdir))
print('Model weights Restored')
tf.logging.info('Model restored!')
train_writer = tf.summary.FileWriter(cfg.logdir + '/train', sess.graph)
for epoch in range(x):
total_num_tr_batch = 0
data_loader = load_training_data(current_epoch)
num_calls = next(data_loader)
print('Training for epoch ' + str(current_epoch + 1) + ': ')
for mini_epoch in range(num_calls):
print('mini_epoch #: {0}'.format(mini_epoch + 1))
trX, trY, num_tr_batch, valX, valY, num_val_batch = next(
data_loader)
total_num_tr_batch += num_tr_batch
avg_loss = []
avg_acc = []
for step in tqdm(range(num_tr_batch),
total=num_tr_batch,
ncols=100,
leave=False,
unit='b'):
start = step * cfg.batch_size
end = start + cfg.batch_size
global_step += 1
_, loss, train_acc, summary_str = sess.run(
[
model.train_op, model.loss, model.accuracy,
model.train_summary
], {
model.image: trX[start:end],
model.labels: trY[start:end]
})
assert not np.isnan(
loss), 'Something wrong! loss is nan...'
if cfg.activate_tensorboard:
train_writer.add_summary(summary_str, global_step)
if (epoch + 1) % cfg.save_freq == 0:
saver.save(
sess, cfg.logdir + '/model_V1_epoch_%04d_step_%02d' %
(epoch + 1, global_step))
return global_step
def train(data_trainset, data_devset):
print('Loading training data...', end=' ')
sys.stdout.flush()
# Load and preprocess the training and validation data
data_loader.load_training_data(data_trainset,
data_devset,
generate_vocab=True)
# Generate the data files in the format required for T2T
os.system('bash ' + os.path.join(config.T2T_DIR, 't2t_datagen_script.sh'))
print('DONE')
print('Training...')
sys.stdout.flush()
# Run the model training
os.system('bash ' + os.path.join(config.T2T_DIR, 't2t_train_script.sh'))
print('DONE')
def test(a):
test_features, test_labels = dl.load_training_data("testing")
test_features = preprocess(test_features)
correct_counter = 0
wrong_counter = 0
for i in range(len(test_features)):
res = a.test_case(test_features[i])
if res == test_labels[i]:
correct_counter += 1
else:
wrong_counter += 1
return getPercentage(correct_counter, wrong_counter)
import network as nw
import data_loader as dl
training_data, testing_data = dl.load_training_data()
print('data loaded')
net = nw.Network([784, 16, 16, 10])
net.stochastic_gradient_descent(training_data, 0.5, 10, 20)
rate = net.test(testing_data)
print(rate)
net.stochastic_gradient_descent(training_data, 0.5, 10, 20)
rate = net.test(testing_data)
print(rate)
net.stochastic_gradient_descent(training_data, 0.5, 10, 20)
rate = net.test(testing_data)
print(rate)
def training():
traing_data = data_loader.load_training_data()
model = svm.SVC(gamma="scale", decision_function_shape="ovo").fit(traing_data[0], traing_data[1])
return model
return test_result
# ------------------------------------------------------------------------------
# top settings
# ------------------------------------------------------------------------------
n_training_data = 10000
n_test_data = 1000
n_epoch = 10
mini_batch_size = 100
learn_rate = 0.005
# ------------------------------------------------------------------------------
# step 1: generate data
# ------------------------------------------------------------------------------
training_data = data_loader.load_training_data(n_training_data)
test_dataset,test_label = data_loader.load_test_data (n_test_data )
# ------------------------------------------------------------------------------
# step 2: setup the model
# ------------------------------------------------------------------------------
label_x = tf.placeholder(tf.float32, [None, 784])
label_y = tf.placeholder(tf.float32, [None, 10])
y_pred=add_layer('output_layer', label_x, 784, 10, tf.nn.softmax)
'''
W = tf.Variable(tf.random_normal([784, 10])); # [1, 784] x [784, 10] = [1, 10]
b = tf.Variable(tf.zeros([10]))
z_pred = tf.matmul(label_x, W) + b
y_pred = tf.nn.softmax(z_pred)
'''
from data_loader import load_training_data, load_reconstructed_training_data, train_test_split
from models import conv_model, dense_model, fit_model, coef_model
from keras import optimizers
raw_x, raw_y = load_training_data('data/raw_maps.npz')
x_train, x_test, y_train, y_test, _, _ = train_test_split(raw_x, raw_y, raw_y)
denses = [5, 20, 50, 100]
neofs = range(1, 30)
epochs = 1000
for hidden_layer_neurons in denses:
model = dense_model(raw_x,
raw_y,
hidden_layer_neurons=hidden_layer_neurons,
name='dense_%i_trained_on_raw_maps' %
hidden_layer_neurons,
optimizer=optimizers.Adam(learning_rate=0.004))
fit_model(x_train, x_test, y_train, y_test, model, epochs=epochs)
model = conv_model(raw_x,
raw_y,
name='conv_trained_on_raw_maps',
optimizer=optimizers.Adam(learning_rate=0.004))
fit_model(x_train, x_test, y_train, y_test, model, epochs=epochs)
for n in neofs:
x, y, pcs, eofs = load_reconstructed_training_data(
'data/reconstructed_maps(neofs=%i).npz' % n)
timer_base = time.time()
if os.path.isfile(model_file_name):
with open(model_file_name, "rb") as f:
xgboost_model = pickle.load(f)
print("loaded dumped model")
else:
if os.path.isfile(dump_file_name):
with open(dump_file_name, "rb") as f:
training_data = pickle.load(f)
trend = pickle.load(f)
print(
f"loaded dumped data, {round(time.time() - timer_base)} seconds elapsed"
)
else:
training_data, trend = data_loader.load_training_data(verbose=True)
with open(dump_file_name, "wb") as f:
pickle.dump(training_data, f)
pickle.dump(trend, f)
print(
f"loaded data from csv file and pickled, {round(time.time() - timer_base)} seconds elapsed"
)
X_train = [item[:-5] for item in training_data]
Y_train = [item[-1] - trend[item[4]] for item in training_data]
parameters = {
"eta": 0.025,
"seed": 0,
"subsample": 0.95,
"colsample_bytree": 0.95,
def main():
parser = get_parser()
args = parser.parse_args()
feature_extractor = FeatureExtractor()
if args.pipeline_type == "analysis":
text_preprocessor = TextPreProcessor(
stop_words_file_path=args.stopwords_file_path)
analyser = DataAnalyser(input_file=args.input_file_path,
text_preprocessor=text_preprocessor)
analyser.get_data_distribution(plot_bar=args.plot_bar)
analyser.get_word_weights(word_thresh=args.word_thresh)
if args.word_cloud:
analyser.generate_word_cloud()
elif args.pipeline_type == "model_selection":
text_preprocessor = TextPreProcessor(
stop_words_file_path=args.stopwords_file_path)
training_data_df = load_training_data(args.train_file_path)
training_data_df["sentence"] = training_data_df["sentence"].map(
text_preprocessor.process)
features = feature_extractor.get_features_for_training(
training_data_df["sentence"], args.vectorizer)
labels = training_data_df["class"]
apply_cross_validation(
features=features,
labels=labels,
k_folds=args.kfolds,
use_svm=args.use_svm,
use_naive_bayes=args.use_naive_bayes,
use_random_forest=args.use_random_forest,
use_logistic_regression=args.use_logistic_regression,
use_xgboost=args.use_xgboost,
use_gradient_boosting=args.use_gradient_boosting,
plot_cv_graph=True,
)
elif args.pipeline_type == "training":
trainer = Trainer(
train_file_path=args.train_file_path,
val_file_path=args.val_file_path,
stop_words_file_path=args.stopwords_file_path,
model_name=args.best_model,
feature_extractor=feature_extractor,
)
training_data_df = load_training_data(args.train_file_path)
trainer.train(
training_data_df,
split_test_size=args.split_size,
vectorizer_name=args.vectorizer,
get_classification_report=args.get_classification_report,
get_confusion_matrix=args.get_confusion_matrix,
)
validation_data_df = load_validation_data(args.val_file_path)
trainer.validate(validation_data_df, vectorizer_name=args.vectorizer)
if args.model_check_point_path:
trainer.save_trained_model(args.model_check_point_path)
elif args.pipeline_type == "prediction":
if not args.stopwords_file_path:
predictor = Predictor()
else:
predictor = Predictor(stop_words_file=args.stopwords_file_path)
if args.input_file_path:
predictor.predict_csv(args.input_file_path, args.output_file_path,
args.model_path)
if args.test_input:
model, vectorizer = predictor.unpickle_the_model(args.model_path)
predictor.predict(args.test_input, model, vectorizer)
def main(options):
train_data_dir = options.train_data_dir
train_files = os.listdir(train_data_dir)
test_data_dir = options.test_data_dir
test_files = os.listdir(test_data_dir)
data = data_loader.load_training_data(train_data_dir, train_files)
images, labels, label_dict, imsize = data
test_data = data_loader.load_test_data(test_data_dir, test_files, label_dict)
test_images, test_labels = test_data
num_classes = len(label_dict)
test_info = test_images, test_labels
has_test_data = False
if len(test_images) > 0:
has_test_data = True
for key in label_dict:
ex_count = np.sum(labels[:, label_dict[key]] == 1.)
print("Number of examples of " + str(key) + ": " + str(ex_count))
print("Feature Mapping: ", label_dict)
print("Image Size: " + str(imsize))
train_info, val_info = data_loader.merge_and_split_data(images, labels)
train_data, train_labels = train_info
val_data, val_labels = val_info
print("Merged and Split Training Data")
print("Training Data Size: ", len(train_data))
print("Validation Data Size: ", len(val_data))
if has_test_data:
for key in label_dict:
ex_count = np.sum(test_labels[:, label_dict[key]] == 1.)
print("Number of test examples of " + str(key) + ": " + str(ex_count))
print("Test Data Size: ", len(test_images))
data_queue = data_queues.QueueManager(train_data, train_labels)
placeholders = utils.create_placeholders(imsize, num_classes)
indata, answer, is_training, keep_prob, learning_rate = placeholders
runnables = vgg_encoder_model.setup_model(indata,
answer,
imsize,
is_training,
keep_prob,
num_classes,
learning_rate)
train_step, loss, predictions, accuracy, summaries, fc_2 = runnables
train_summary, val_summary, test_summary = summaries
init = tf.global_variables_initializer()
print("Setup Model")
log_dir = options.tb_log_dir
features_dir = options.features_dir
with tf.device('/gpu:0'):
with tf.Session() as sess:
train_writer = tf.summary.FileWriter(log_dir + "/train", sess.graph)
val_writer = tf.summary.FileWriter(log_dir + "/val", sess.graph)
test_writer = tf.summary.FileWriter(log_dir + "/test", sess.graph)
sess.run(init)
patience = 0
max_patience = 1000 # wait for 30 steps of val loss not decreasing
min_val_loss = float("inf")
epoch = 0
while patience < max_patience:
epoch += 1
print("EPOCH: ", str(epoch))
start = time.time()
train_statistics = model_runner.process_train_data(data_queue,
placeholders,
runnables,
train_writer,
num_classes,
sess)
avg_train_loss, train_acc = train_statistics[:2]
train_confusion_matrix = train_statistics[2]
train_misclassified = train_statistics[3]
end = time.time()
val_statistics = model_runner.process_data(val_info,
placeholders,
runnables,
val_writer,
num_classes,
False,
sess)
avg_val_loss, val_acc = val_statistics[:2]
val_confusion_matrix = val_statistics[2]
val_misclassified = val_statistics[3]
val_features = val_statistics[4]
if has_test_data:
test_statistics = model_runner.process_data(test_info,
placeholders,
runnables,
test_writer,
num_classes,
True,
sess)
avg_test_loss, test_acc = test_statistics[:2]
test_confusion_matrix = test_statistics[2]
test_misclassified = test_statistics[3]
test_features = test_statistics[4]
print("train_loss: " + str(avg_train_loss) + " val_loss: " + str(avg_val_loss) +
" test_loss: " + str(avg_test_loss))
else:
print("train_loss: " + str(avg_train_loss) + " val_loss: " + str(avg_val_loss))
print("train_acc: " + str(train_acc))
print("val_acc: " + str(val_acc))
if has_test_data:
print("test_acc: ", str(test_acc))
print("Training Time: ", str(end - start))
if avg_val_loss < min_val_loss:
min_val_loss = avg_val_loss
print("Training Confusion:\n", train_confusion_matrix)
print("Validation Confusion:\n", val_confusion_matrix)
if has_test_data:
print("Test Confusion:\n", test_confusion_matrix)
utils.store_misclassified(train_misclassified,
val_misclassified,
test_misclassified=test_misclassified)
else:
utils.store_misclassified(train_misclassified,
val_misclassified)
pickle_name = [k + "_{}".format(v) for k, v in label_dict.items()]
pickle_name = '_'.join(pickle_name)
pickle_name = pickle_name + ".p"
final_features = {'val': val_features}
if has_test_data:
final_features['test'] = test_features
with open(os.path.join(features_dir, pickle_name), 'wb') as outf:
pickle.dump(final_features, outf, protocol=pickle.HIGHEST_PROTOCOL)
patience = 0
else:
patience += 1
def main(nb_epoch=1,
data_augmentation=True,
noise=True,
maxout=True,
dropout=True,
l1_reg=False,
l2_reg=True):
# l1 and l2 regularization shouldn't be true in the same time
if l1_reg and l2_reg:
print("No need to run l1 and l2 regularization in the same time")
quit()
# print settings for this experiment
print("number of epoch: {0}".format(nb_epoch))
print("data augmentation: {0}".format(data_augmentation))
print("noise: {0}".format(noise))
print("maxout: {0}".format(maxout))
print("dropout: {0}".format(dropout))
print("l1: {0}".format(l1_reg))
print("l2: {0}".format(l2_reg))
# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
# split the validation dataset
X_train, X_valid, y_train, y_valid = train_test_split(X_train,
y_train,
test_size=0.2,
random_state=0)
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_valid.shape[0], 'valid samples')
print(X_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_valid = np_utils.to_categorical(y_valid, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
X_train = X_train.astype('float32')
X_valid = X_valid.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_valid /= 255
X_test /= 255
##### try loading data using data_loader.py ####
data_loader.download_and_extract(data_path, data_url)
class_names = data_loader.load_class_names()
print(class_names)
images_train, cls_train, labels_train = data_loader.load_training_data()
images_test, cls_test, labels_test = data_loader.load_test_data()
X_train, Y_train = images_train, labels_train
X_test, Y_test = images_test, labels_test
X_train, X_valid, Y_train, Y_valid = train_test_split(X_train,
Y_train,
test_size=0.2,
random_state=0)
print("Size of:")
print("- Training-set:\t\t{}".format(len(X_train)))
print("- Validation-set:\t\t{}".format(len(X_valid)))
print("- Test-set:\t\t{}".format(len(X_test)))
model = Sequential()
if noise:
model.add(
GaussianNoise(sigma,
input_shape=(img_channels, img_rows, img_cols)))
model.add(
Convolution2D(32,
3,
3,
border_mode='same',
input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
# model.add(MaxPooling2D(pool_size=(2, 2)))
if dropout:
model.add(Dropout(0.25))
model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
# model.add(MaxPooling2D(pool_size=(2, 2)))
if dropout:
model.add(Dropout(0.25))
model.add(Flatten())
if maxout:
model.add(MaxoutDense(512, nb_feature=4, init='glorot_uniform'))
else:
if not (l1_reg or l2_reg):
model.add(Dense(512))
# activation regularization not implemented yet
if l1_reg:
model.add(Dense(512, W_regularizer=l1(l1_weight)))
elif l2_reg:
model.add(Dense(512, W_regularizer=l2(l2_weight)))
model.add(Activation('relu'))
if dropout:
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
start_time = time.time()
if not data_augmentation:
his = model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data=(X_valid, Y_valid),
shuffle=True)
else:
# this will do preprocessing and realtime data augmentation
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=True, # apply ZCA whitening
rotation_range=
0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(X_train)
# fit the model on the batches generated by datagen.flow()
his = model.fit_generator(datagen.flow(X_train,
Y_train,
batch_size=batch_size),
samples_per_epoch=X_train.shape[0],
nb_epoch=nb_epoch,
validation_data=(X_valid, Y_valid))
# evaluate our model
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
print('training time', time.time() - start_time)
# wirte test accuracy to a file
output_file_name = './output_l1l2/train_val_loss_with_dropout_epochs_{0}_data_augmentation_{1}_noise_{2}_maxout_{3}_dropout_{4}_l1_{5}_l2_{6}_sigma_{7}_l1weight_{8}_l2weight_{9}.txt'.format(
nb_epoch, data_augmentation, noise, maxout, dropout, l1_reg, l2_reg,
sigma, l1_weight, l2_weight)
print(output_file_name)
with open(output_file_name, "w") as text_file:
text_file.write('Test score: {}'.format(score[0]))
text_file.write('\n')
text_file.write('Test accuracy: {}'.format(score[1]))
text_file.close()
# visualize training history
train_loss = his.history['loss']
val_loss = his.history['val_loss']
plt.plot(range(1,
len(train_loss) + 1),
train_loss,
color='blue',
label='train loss')
plt.plot(range(1,
len(val_loss) + 1),
val_loss,
color='red',
label='val loss')
plt.legend(loc="upper left", bbox_to_anchor=(1, 1))
plt.xlabel('#epoch')
plt.ylabel('loss')
# @TODO what's the deal around here ~"~"?
output_fig_name = './output_no_maxout/train_val_loss_with_dropout_epochs_{0}_data_augmentation_{1}_noise_{2}_maxout_{3}_dropout_{4}_l1_{5}_l2_{6}_sigma_{7}_l1weight_{8}_l2weight_{9}.png'.format(
nb_epoch, data_augmentation, noise, maxout, dropout, l1_reg, l2_reg,
sigma, l1_weight, l2_weight)
plt.savefig(output_fig_name, dpi=300)
plt.show()
