def fit(self, X, y, max_epochs=20, validation_split=0.1):
seq_len = X.shape[1]
self.build_train_model(seq_len)
split_idx = int((1. - validation_split) * X.shape[0])
X, X_val = X[:split_idx], X[split_idx:]
y, y_val = y[:split_idx], y[split_idx:]
checkpoint = ModelCheckpoint("best_weights.h5",
monitor='val_loss',
save_best_only=True,
verbose=1)
early_stop = EarlyStopping(monitor='val_loss', patience=150, verbose=1)
callbacks = [checkpoint, early_stop]
if self.monitor:
monitor = RemoteMonitor(root='http://localhost:9000')
callbacks = callbacks + [monitor]
try:
self.train_model.fit(X,
y,
nb_epoch=max_epochs,
validation_data=(X_val, y_val),
callbacks=callbacks)
except KeyboardInterrupt:
logger.info(
"Training interrupted! Restoring best weights and saving..")
self.train_model.load_weights("best_weights.h5")
self._weights_updated = True
self.save()
def train(self, X, y, validation_split=0.1, max_epochs=100):
n_samples = X.shape[0]
seq_len = X.shape[1]
if self.model is None:
self.model = self.build_model(seq_len=seq_len)
split_idx = int((1. - validation_split) * n_samples)
X_train, X_val = X[:split_idx], X[split_idx:]
y_train, y_val = y[:split_idx], y[split_idx:]
checkpoint = ModelCheckpoint("best_anomaly_weights.h5",
monitor='val_acc',
save_best_only=True,
verbose=1)
early_stop = EarlyStopping(monitor='val_acc', patience=150, verbose=1)
callbacks = [checkpoint, early_stop]
if self.monitor:
monitor = RemoteMonitor(root='http://localhost:9000')
callbacks = callbacks + [monitor]
try:
logger.info("Beginning anomaly detector training..")
self.model.fit([X_train],
y_train,
nb_epoch=max_epochs,
validation_data=([X_val], y_val),
callbacks=callbacks)
except KeyboardInterrupt:
logger.info(
"Training interrupted! Restoring best weights and saving..")
self.model.load_weights("best_anomaly_weights.h5")
self.save()
def __init__(self, model_obj):
self.model_obj = model_obj
self.checkpointer = ModelCheckpoint(
filepath='/Users/rishab/Desktop/fynd/solid_app/tmp/weights.hdf5',
verbose=1,
save_best_only=True)
self.progbarLogger = ProgbarLogger(count_mode='steps')
self.early_monitor = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='auto')
self.remote_monitor = RemoteMonitor(root='http://0.0.0.0:5001',
path='/publish',
field='data',
headers=None)
self.history = LossHistory()
self.board = TensorBoard(log_dir='./logs',
histogram_freq=0,
batch_size=32,
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None)
def get_callbacks(job):
fld.create_fld_if_not_exist(fld.get_path(fld.model_scoring,
job.job_name,
'model_ckeckpoint'))
m_file_name = 'model_ckeckpoint_'+ \
datetime.datetime.now().strftime("%Y-%d-%m-%H-%M")\
+'-{epoch:04d}-{val_acc:.4f}.hdf5'
m_file_name = 'model_ckeckpoint.hdf5'
path = fld.get_path(fld.model_scoring, job.job_name,
'model_ckeckpoint', m_file_name)
ckpt = ModelCheckpoint(path, monitor='val_loss', verbose=0,
save_best_only=True, save_weights_only=False,
mode='auto')
es = EarlyStopping(monitor='val_loss', min_delta=0,
patience=10, verbose=0, mode='auto')
lr_plateau = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=5, min_lr=0.000001)
fld.create_fld_if_not_exist(fld.get_path(fld.model_scoring,
job.job_name,
'RemoteMonitor'))
path_rm = fld.get_path(fld.model_scoring, job.job_name,
'RemoteMonitor')
rm = RemoteMonitor(path = path_rm)
try:
imp.find_module('tensorflow')
fld.create_fld_if_not_exist(fld.get_path(fld.model_scoring,
job.job_name, 'TensorBoard'))
path_tb = fld.get_path(fld.model_scoring, job.job_name, 'TensorBoard')
tb = TensorBoard(log_dir=path_tb, histogram_freq=0, write_graph=True,
write_images=False)
callback_lst = [ckpt, es, lr_plateau, rm, tb]
except:
callback_lst = [ckpt, es, lr_plateau, rm]
return callback_lst
def cnn_model():
(x_train, y_train), _ = mnist.load_data()
# 归一化
x_train = x_train.reshape(-1, 28, 28, 1) / 255.
# one-hot
y_train = np_utils.to_categorical(y=y_train, num_classes=10)
model = Sequential([
# input_shape:输入平面，就在第一个位置设置
# filters：卷积核、滤波器
# kernel_size：卷积核大小
# strides：步长
# padding有两种方式：same/valid
# activation：激活函数
Convolution2D(input_shape=(28, 28, 1), filters=32, kernel_size=5, strides=1, padding='same', activation=relu),
MaxPool2D(pool_size=2, strides=2, padding='same'),
Convolution2D(filters=64, kernel_size=5, padding='same', activation=relu),
MaxPool2D(pool_size=2, trainable=2, padding='same'),
Flatten(), # 扁平化
Dense(units=1024, activation=relu),
Dropout(0.5),
Dense(units=10, activation=softmax),
])
opt = Adam(lr=1e-4)
model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=['accuracy'])
model.fit(x=x_train, y=y_train, batch_size=64, epochs=20, callbacks=[RemoteMonitor()])
model_save(model, './model.h5')
def fit(self, inputs, target, max_epochs=10, validation_split=0.1):
n_sequences = target.shape[0]
seq_len = target.shape[1]
data_dim = target.shape[2]
assert self.data_dim == data_dim
# Build the train model
list_in = inputs[:]
if not self.with_trending_prior:
list_in.append(
STORNPriorModel.standard_input(n_sequences, seq_len,
self.latent_dim))
self.train_model = self._build(Phases.train, seq_shape=seq_len)
# self.train_model.load_weights("start_weights.h5")
# Do a validation split of all the inputs
split_idx = int((1. - validation_split) * n_sequences)
train_input, valid_input = [
list(t)
for t in zip(*[(X[:split_idx], X[split_idx:]) for X in list_in])
]
train_target, valid_target = target[:split_idx], target[split_idx:]
checkpoint = ModelCheckpoint("best_storn_weights.h5",
monitor='val_loss',
save_best_only=True,
verbose=1)
early_stop = EarlyStopping(monitor='val_loss', patience=25, verbose=1)
try:
# A workaround so that keras does not complain about target and pred shape mismatches
padded_target = np.concatenate(
(train_target,
np.zeros((train_target.shape[0], seq_len,
4 * self.latent_dim + data_dim))),
axis=-1)
padded_valid_target = np.concatenate(
(valid_target,
np.zeros((valid_target.shape[0], seq_len,
4 * self.latent_dim + self.data_dim))),
axis=-1)
callbacks = [checkpoint, early_stop]
if self.monitor:
monitor = RemoteMonitor(root='http://localhost:9000')
callbacks = callbacks + [monitor]
self.train_model.fit(train_input,
padded_target,
validation_data=(valid_input,
[padded_valid_target]),
callbacks=callbacks,
nb_epoch=max_epochs)
except KeyboardInterrupt:
logger.info(
"Training interrupted! Restoring best weights and saving..")
self.train_model.load_weights("best_storn_weights.h5")
self._weights_updated = True
self.save()
def main(model):
print('Loading model')
nvidia = model
nvidia.compile(optimizer=Adam(lr=0.0009,
clipnorm=.25,
beta_1=0.7,
beta_2=0.99),
loss='mse',
metrics=['acc'])
checkpointer = ModelCheckpoint(filepath="{epoch:02d}-{val_loss:.12f}.hdf5",
verbose=1,
save_best_only=True)
lr_plateau = ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=5,
min_lr=0.000001,
verbose=1,
mode=min)
monitor = RemoteMonitor(root='http://localhost:9000',
path='/publish/epoch/end/',
field='data',
headers=None)
epochs = 20
batch_size = 32
print('Starting training')
history = nvidia.fit(train_x,
train_y,
validation_data=(test_x, test_y),
nb_epoch=epochs,
batch_size=batch_size,
callbacks=[checkpointer, lr_plateau, monitor])
print('Done')
hist = pd.DataFrame(history.history)
plt.figure(figsize=(12, 12))
plt.plot(hist["loss"])
plt.plot(hist["val_loss"])
plt.plot(hist["acc"])
plt.plot(hist["val_acc"])
plt.legend()
plt.show()
return history, nvidia
def rnn_model():
(x_train, y_train), _ = mnist.load_data()
# 归一化
x_train = x_train / 255.
# one-hot
y_train = np_utils.to_categorical(y=y_train, num_classes=10)
model = Sequential([
SimpleRNN(units=50, input_shape=(28, 28)),
Dense(units=10, activation=softmax),
])
opt = RMSprop(lr=1e-4)
model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=['accuracy'])
model.fit(x=x_train, y=y_train, batch_size=64, epochs=20, callbacks=[RemoteMonitor()])
model_save(model, './model.h5')
def nn_model():
(x_train, y_train), _ = mnist.load_data()
# 归一化
x_train = x_train.reshape(x_train.shape[0], -1) / 255.
# one-hot
y_train = np_utils.to_categorical(y=y_train, num_classes=10)
# constant(value=1.)自定义常数，constant(value=1.)===one()
# 创建模型:输入784个神经元，输出10个神经元
model = Sequential([
Dense(units=200, input_dim=784, bias_initializer=constant(value=1.), activation=tanh),
Dense(units=100, bias_initializer=one(), activation=tanh),
Dense(units=10, bias_initializer=one(), activation=softmax),
])
opt = SGD(lr=0.2, clipnorm=1.) # 优化器
model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=['acc', 'mae']) # 编译
model.fit(x_train, y_train, batch_size=64, epochs=20, callbacks=[RemoteMonitor()])
model_save(model, './model.h5')
def train(self, image_generator, epochs=[20]):
self.model_cache = {}
remote = RemoteMonitor(root='https://localhost:9000')
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
verbose=1,
patience=3,
min_lr=0.001)
checkpointer = ModelCheckpoint(filepath=self.checkpoint,
verbose=1,
save_best_only=False)
for i, nb_epoch in enumerate(epochs):
print("ERA {}: ".format(i))
X_train, y_train = image_generator.data(mode='train')
X_val, y_val = image_generator.data(mode='val')
print(X_train.shape, y_train.shape)
print(y_val.sum(axis=0))
np.savez("results/train_data_era" + str(i) + ".npz",
X_train=X_train,
y_train=y_train,
X_val=X_val,
y_val=y_val)
#IPython.embed();
try:
self.local_model.fit(X_train,
y_train,
epochs=nb_epoch,
batch_size=image_generator.batch_size,
verbose=1,
validation_data=(X_val, y_val),
callbacks=[reduce_lr, checkpointer])
preds = self.local_model.predict(X_val)
except KeyboardInterrupt:
pass
del X_train, y_train, X_val, y_val
image_generator.checkpoint(self)
def train_model():
if cxl_model:
embedding_matrix = load_embedding()
else:
embedding_matrix = {}
train, label = vocab_train_label(train_path,
vocab=vocab,
tags=tag,
max_chunk_length=length)
n = np.array(label, dtype=np.float)
labels = n.reshape((n.shape[0], n.shape[1], 1))
model = Sequential([
Embedding(input_dim=len(vocab),
output_dim=300,
mask_zero=True,
input_length=length,
weights=[embedding_matrix],
trainable=False),
SpatialDropout1D(0.2),
Bidirectional(layer=LSTM(units=150,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)),
TimeDistributed(Dense(len(tag), activation=relu)),
])
crf_ = CRF(units=len(tag), sparse_target=True)
model.add(crf_)
model.compile(optimizer=Adam(),
loss=crf_.loss_function,
metrics=[crf_.accuracy])
model.fit(x=np.array(train),
y=labels,
batch_size=16,
epochs=4,
callbacks=[RemoteMonitor()])
model.save(model_path)
def main(data_module,
model_module,
optimizer_module,
filename,
config,
use_val=False):
"""Patch everything together."""
batch_size = config['train']['batch_size']
nb_epoch = config['train']['epochs']
today = datetime.datetime.now()
datestring = today.strftime('%Y%m%d-%H%M-%S')
# The data, shuffled and split between train and test sets:
data = data_module.load_data(config)
print("Data loaded.")
X_train, y_train = data['x_train'], data['y_train']
X_train = data_module.preprocess(X_train)
if 'use_val' in config['train']:
use_val = config['train']['use_val']
use_val = True
if use_val:
X_test, y_test = data['x_val'], data['y_val']
else:
X_test, y_test = data['x_test'], data['y_test']
X_test = data_module.preprocess(X_test)
# load hierarchy, if present
if 'hierarchy_path' in config['dataset']:
ret = handle_hierarchies(config, data_module, X_train, y_train, X_test,
y_test)
# hierarchy = ret['hierarchy']
X_train = ret['X_train']
y_train = ret['y_train']
X_test = ret['X_test']
y_test = ret['y_test']
nb_classes = data_module.n_classes
logging.info("# classes = {}".format(data_module.n_classes))
img_rows = data_module.img_rows
img_cols = data_module.img_cols
img_channels = data_module.img_channels
da = config['train']['data_augmentation']
# Convert class vectors to binary class matrices.
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
# Y_train = Y_train.reshape((-1, 1, 1, nb_classes)) # For fcn
# Y_test = Y_test.reshape((-1, 1, 1, nb_classes))
if 'smooth_train' in config['dataset']:
Y_train = np.load(config['dataset']['smooth_train'])
if 'smooth_test_path' in config['dataset']:
Y_test = np.load(config['dataset']['smooth_test_path'])
# Input shape depends on the backend
if K.image_dim_ordering() == "th":
input_shape = (img_channels, img_rows, img_cols)
else:
input_shape = (img_rows, img_cols, img_channels)
model = model_module.create_model(nb_classes, input_shape, config)
print("Model created")
if 'initializing_model_path' in config['model']:
init_model_path = config['model']['initializing_model_path']
if not os.path.isfile(init_model_path):
logging.error(
"initializing_model={} not found".format(init_model_path))
sys.exit(-1)
init_model = load_model(init_model_path)
layer_dict_init = dict([(layer.name, layer)
for layer in init_model.layers])
layer_dict_model = dict([(layer.name, layer)
for layer in model.layers])
for layer_name in layer_dict_model.keys():
if layer_name in layer_dict_init:
print("\tLoad layer weights '{}'".format(layer_name))
weights = layer_dict_init[layer_name].get_weights()
try:
layer_dict_model[layer_name].set_weights(weights)
except ValueError:
print("\t\twrong shape - skip")
logging.info("Done initializing")
model.summary()
optimizer = optimizer_module.get_optimizer(config)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=["accuracy"])
print("Finished compiling")
print("Building model...")
checkpoint_fname = os.path.basename(config['train']['artifacts_path'])
if 'saveall' in config['train'] and config['train']['saveall']:
checkpoint_fname = ("{}_{}.chk.{{epoch:02d}}.h5".format(
checkpoint_fname, datestring))
save_best_only = False
else:
checkpoint_fname = "{}_{}.chk.h5".format(checkpoint_fname, datestring)
save_best_only = True
model_chk_path = os.path.join(config['train']['artifacts_path'],
checkpoint_fname)
model_chk_path = get_nonexistant_path(model_chk_path)
checkpoint = ModelCheckpoint(model_chk_path,
monitor="val_acc",
save_best_only=save_best_only,
save_weights_only=False)
history_cb = History()
callbacks = [checkpoint, history_cb] # remote,
if 'tensorboard' in config['train'] and config['train']['tensorboard']:
tensorboard = TensorBoard(log_dir='./logs',
histogram_freq=0,
write_graph=True,
write_images=True)
callbacks.append(tensorboard)
if 'remote' in config['train'] and config['train']['remote']:
remote = RemoteMonitor(root='http://localhost:9000')
callbacks.append(remote)
if 'lr_reducer' in config['train'] and config['train']['lr_reducer']:
lr_reducer = ReduceLROnPlateau(monitor='val_acc',
factor=0.3,
cooldown=0,
patience=3,
min_lr=0.5e-6,
verbose=1)
callbacks.append(lr_reducer)
if 'clr' in config['train']:
clr = CyclicLR(base_lr=config['train']['clr']['base_lr'],
max_lr=config['train']['clr']['max_lr'],
step_size=(config['train']['clr']['step_size'] *
(X_train.shape[0] // batch_size)),
mode=config['train']['clr']['mode'])
callbacks.append(clr)
X_train = np.append(X_train, X_test, axis=0)
Y_train = np.append(Y_train, Y_test, axis=0)
if not da:
print('Not using data augmentation.')
model.save(model_chk_path.format(epoch=0).replace('.00.', '.00.a.'))
t0 = time.time()
model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=nb_epoch,
validation_data=(X_test, Y_test),
shuffle=True,
callbacks=callbacks)
t1 = time.time()
t2 = t1
epochs_augmented_training = 0
else:
print('Using real-time data augmentation.')
if 'hue_shift' in da:
hsv_augmentation = (da['hue_shift'], da['saturation_scale'],
da['saturation_shift'], da['value_scale'],
da['value_shift'])
else:
hsv_augmentation = None
# This will do preprocessing and realtime data augmentation:
datagen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=da['featurewise_center'],
# set each sample mean to 0
samplewise_center=da['samplewise_center'],
# divide inputs by std of the dataset
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=da['samplewise_std_normalization'],
zca_whitening=da['zca_whitening'],
# randomly rotate images in the range (degrees, 0 to 180)
rotation_range=da['rotation_range'],
# randomly shift images horizontally (fraction of total width)
width_shift_range=da['width_shift_range'],
# randomly shift images vertically (fraction of total height)
height_shift_range=da['height_shift_range'],
horizontal_flip=da['horizontal_flip'],
vertical_flip=da['vertical_flip'],
hsv_augmentation=hsv_augmentation,
zoom_range=da['zoom_range'],
shear_range=da['shear_range'],
channel_shift_range=da['channel_shift_range'])
# Compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied).
datagen.fit(X_train, seed=0)
# Apply normalization to test data
for i in range(len(X_test)):
X_test[i] = datagen.standardize(X_test[i])
# Fit the model on the batches generated by datagen.flow().
steps_per_epoch = X_train.shape[0] // batch_size
model.save(model_chk_path.format(epoch=0).replace('.00.', '.00.a.'))
t0 = time.time()
model.fit_generator(datagen.flow(X_train,
Y_train,
batch_size=batch_size),
steps_per_epoch=steps_per_epoch,
epochs=nb_epoch,
validation_data=(X_test, Y_test),
callbacks=callbacks)
t1 = time.time()
# Train one epoch without augmentation to make sure data distribution
# is fit well
loss_history = history_cb.history["loss"]
epochs_augmented_training = len(loss_history)
model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=nb_epoch,
validation_data=(X_test, Y_test),
shuffle=True,
callbacks=callbacks,
initial_epoch=len(loss_history))
t2 = time.time()
loss_history = history_cb.history["loss"]
acc_history = history_cb.history["acc"]
val_acc_history = history_cb.history["val_acc"]
np_loss_history = np.array(loss_history)
np_acc_history = np.array(acc_history)
np_val_acc_history = np.array(val_acc_history)
history_data = zip(list(range(1,
len(np_loss_history) + 1)), np_loss_history,
np_acc_history, np_val_acc_history)
history_data = [(el[0], "%0.4f" % el[1], "%0.4f" % el[2], "%0.4f" % el[3])
for el in history_data]
history_fname = os.path.basename(config['train']['artifacts_path'])
history_fname = "{}_{}_history.csv".format(history_fname, datestring)
csv_path = os.path.join(config['train']['artifacts_path'], history_fname)
csv_path = get_nonexistant_path(csv_path)
with open(csv_path, 'w') as fp:
writer = csv.writer(fp, delimiter=',')
writer.writerows([("epoch", "loss", "acc", "val_acc")])
writer.writerows(history_data)
training_time = t1 - t0
readjustment_time = t2 - t1
print("wall-clock training time: {}s".format(training_time))
model_fn = os.path.basename(config['train']['artifacts_path'])
model_fn = "{}_{}.h5".format(model_fn, datestring)
model_fn = os.path.join(config['train']['artifacts_path'], model_fn)
model_fn = get_nonexistant_path(model_fn)
model.save(model_fn)
# Store training meta data
data = {
'training_time': training_time,
'readjustment_time': readjustment_time,
'HOST': platform.node(),
'epochs': len(history_data),
'epochs_augmented_training': epochs_augmented_training,
'config': config
}
meta_train_fname = os.path.join(config['train']['artifacts_path'],
"train-meta_{}.json".format(datestring))
meta_train_fname = get_nonexistant_path(meta_train_fname)
with open(meta_train_fname, 'w') as outfile:
str_ = json.dumps(data,
indent=4,
sort_keys=True,
separators=(',', ': '),
ensure_ascii=False)
outfile.write(str_)
import pandas as pd
import numpy as np
np.random.seed(1337) # for reproducibility
from keras import backend as K
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.callbacks import Callback, RemoteMonitor
from keras.utils import np_utils
# enable multi-CPU
#import theano
#theano.config.openmp = True
monitor = RemoteMonitor(root='http://localhost:9000')
# input image dimensions
img_rows, img_cols = 28, 28
batch_size = 128 # Number of images used in each optimization step
nb_classes = 62 # One class per digit/lowercase letter/uppercase letter
nb_epoch = 70 # Number of times the whole data is used to learn
# Read the train and test datasets
train = pd.read_csv("emnist/train.csv").values
test = pd.read_csv("emnist/test.csv").values
print('train shape:', train.shape)
print('test shape:', test.shape)
# checkpointer = ModelCheckpoint('weights.{epoch:02d}-{val_loss:.2f}.hdf5', save_best_only=True)
# logger = ProgbarLogger(count_mode='samples')
# remote = RemoteMonitor(root='192.168.0.157:9000')
print('Model compiled, start training...')
# model.fit(training[0], training[1], epochs=250, batch_size=1, validation_data=validation, callbacks=[
# ModelCheckpoint('weights.{epoch:02d}-{val_loss:.2f}.hdf5', save_best_only=True),
# ProgbarLogger(count_mode='samples'),
# RemoteMonitor(root='192.168.0.157:9000')
# ])
model.fit_generator(generate_batch(training), steps_per_epoch=248, epochs=500 , validation_data=generate_batch(validation), validation_steps=248, callbacks=[
ModelCheckpoint('weights.{epoch:02d}-{val_loss:.2f}.hdf5', save_best_only=True),
RemoteMonitor(root='192.168.0.157:9000')
])
# input_0 = Input(shape=(2,), name='input')
# path=Dense(4, activation='relu')(input_0)
# out=Dense(1, activation='linear')(path)
# model=Model(inputs=input_0, outputs=out)
# model.compile(optimizer='nadam', loss='mean_squared_error')
#
# in_data = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
# out_data = np.array([0, 1, 1, 0])
# logger.set_model(model)
# remote.set_model(model)
#
# model.fit(in_data, out_data, epochs=25, batch_size=2)
# get_3rd_layer_output = backend.function([model.layers[0].input], [model.layers[3].output])
# layer_output = get_3rd_layer_output([test_generator.next()[0]])[0]
# print layer_output[0][0:3].transpose().shape
# image = Image.fromarray(layer_output[0][0:3].transpose())
# image.save("test.png")
callbacks = []
if args.stop:
callbacks.append(
EarlyStopping(monitor="val_loss",
min_delta=0.001,
patience=5,
mode="auto"))
if args.visualize:
callbacks.append(RemoteMonitor(root="http://localhost:9000"))
try:
model.fit_generator(train_generator,
train_generator.nb_sample,
args.epochs,
validation_data=test_generator,
nb_val_samples=train_generator.nb_sample / 5,
callbacks=callbacks,
class_weight=args.weight)
print "\x07"
except Exception, e:
print "Error during training:"
print str(e)
for i in range(3):
print "\x07"
print('Loading model')
nvidia = model()
checkpointer = ModelCheckpoint(filepath="{epoch:02d}-{val_loss:.12f}.hdf5",
verbose=1,
save_best_only=True)
lr_plateau = ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=5,
min_lr=0.000001,
verbose=1,
mode=min)
monitor = RemoteMonitor(root='http://localhost:9000',
path='/publish/epoch/end/',
field='data',
headers=None)
epochs = 100
batch_size = 64
print('Starting training')
history = nvidia.fit(train_X,
train_Y,
validation_data=(test_X, test_Y),
nb_epoch=epochs,
batch_size=batch_size,
callbacks=[checkpointer, lr_plateau, monitor])
print('Done')
hist = pd.DataFrame(history.history)
from keras.layers import LSTM
from keras.datasets import imdb
from keras.preprocessing.text import Tokenizer
from keras import optimizers
import numpy as np
import keras, gensim, logging
from keras.callbacks import RemoteMonitor
from gensim_training import WordTrainer
import os
from os import listdir
from os.path import isfile, join
from keras.utils import plot_model
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO)
remote = RemoteMonitor()
MAX_SEQUENCE_LENGTH = 1000
MAX_NB_WORDS = 750
EMBEDDING_DIM = 10
VALIDATION_SPLIT = 0.2
max_features = 750
maxlen = 150 # cut texts after this number of words (among top max_features most common words)
batch_size = 512
num_classes = 4
print('Loading data...')
ds = Dataset()
X_all = ds.X_all_sent
Y_all = ds.Y_all
print('loaded word2vec...')
wordTrainer = WordTrainer()
def train_data(data, name):
data = np.array(data)
scaler = sk.MinMaxScaler(feature_range=(0, 1))
scaled_data = scaler.fit_transform(data)
scale = scaler.scale_[0]
min = scaler.data_min_[0]
train_data, train_label, test_data, test_label = \
generate_keras_data(scaled_data, input_timestep, output_timestep)
model = Sequential()
model.add(
CuDNNGRU(units=units,
input_shape=(train_data.shape[1], train_data.shape[2]),
return_sequences=True))
model.add(Dropout(0.25))
model.add(CuDNNGRU(units=units, return_sequences=False))
model.add(Dropout(0.3))
model.add(Dense(units=train_label.shape[1]))
model.compile(optimizer='adam', loss='mse', metrics=['accuracy', 'mae'])
model.summary()
history = History()
checkpoint = ModelCheckpoint('data/sentiment_analysis/results/' + name +
'_{epoch:02d}.hdf5',
monitor='val_acc',
verbose=0,
period=5)
remote = RemoteMonitor()
csv_log = CSVLogger('data/sentiment_analysis/results/' + name + '.log',
separator=',')
tensorboard = TensorBoard(log_dir='data/sentiment_analysis/results/' +
name + '_logs',
histogram_freq=2,
write_grads=True)
model.fit(train_data,
train_label,
batch_size=32,
epochs=5,
verbose=1,
validation_split=0.1,
callbacks=[history, checkpoint, remote, csv_log, tensorboard])
print(history.history)
with open('data/sentiment_analysis/results/history_' + name + '.pkl',
'wb') as hist_file:
pickle.dump(history.history, hist_file)
nn_pred = model.predict(test_data)
true_label = (test_label / scale) + min
pred_label = (nn_pred / scale) + min
past_range = len(data) - len(pred_label)
plt.plot(range(past_range - 1000, past_range),
data[past_range - 1000:past_range, 0],
'y',
label='Past values')
plt.plot(range(past_range, len(data)),
true_label[:, 1],
'g',
label='True values')
plt.plot(range(past_range, len(data)),
pred_label[:, 1],
'r',
label='Predicted values')
plt.xlabel('Time')
plt.ylabel('Price')
plt.legend(loc=0)
plt.title('Prediction and true values')
plt.savefig('data/sentiment_analysis/results/' + name + '_input_' +
str(input_timestep) + 'h')
plt.show()
plt.close()
average_sign = []
correct_sign = []
mse = []
mae = []
for i in range(0, pred_label.shape[0]):
past_label = test_data[i, :, :]
past_label = scaler.inverse_transform(past_label)
past_label = past_label[:, 0]
sign_current = 0
sign = 0
for j in range(pred_label.shape[1]):
sign_current = np.sign(pred_label[i, j] - past_label[-1])
if sign_current == np.sign(true_label[i, j] - past_label[-1]):
sign = 1
else:
sign = 0
mse_current = np.mean((pred_label[i, 1] - true_label[i, 1])**2)
mse.append(mse_current)
mae_current = np.absolute(pred_label[i, 1] - true_label[i, 1])
mae.append(mae_current)
average_sign.append(sign_current)
correct_sign.append(sign)
average_sign = np.mean(average_sign)
correct_sign = np.mean(correct_sign)
mse = np.mean(mse)
mae = np.mean(mae)
return average_sign, correct_sign, mse, mae
samples_test_count = test_loader.samples_count
print "Creating model...."
model = Sequential()
model.add(
LSTM(512, input_shape=(5, 512), dropout_U=0.5, return_sequences=True))
model.add(Dropout(0.5))
model.add(LSTM(250, dropout_U=0.5, dropout_W=0.5))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
print "Loading data..."
X_train, Y_train = loader_train.load_sequence_samples(
num_elements=samples_train_count)
X_test, Y_test = test_loader.load_sequence_samples(
num_elements=samples_test_count)
model_check_pointing = ModelCheckpoint(
'../models/LSTM_weights.{epoch:03d}-{val_loss:.4f}.hdf5',
monitor='val_loss',
verbose=1,
save_best_only=False,
mode='auto')
rmm = RemoteMonitor(root='http://localhost:8080')
model.fit(X_train,
Y_train,
batch_size=16,
nb_epoch=2000,
verbose=2,
validation_data=(X_test, Y_test),
callbacks=[model_check_pointing, rmm])
regressor.compile(optimizer='adam', loss='mean_squared_error')
mcp = ModelCheckpoint('lstm_weights/weights{epoch:04d}.h5',
save_weights_only=True,
period=5)
tb = TensorBoard('logs')
#es = EarlyStopping(monitor='val_loss', min_delta=1e-10, patience=10, verbose=1)
rlr = ReduceLROnPlateau(monitor='val_loss',
factor=0.8,
patience=10,
verbose=1,
min_lr=0.0005)
remote = RemoteMonitor(root='http://0.0.0.0:9000',
path='/publish/epoch/end/',
field='data',
headers=None,
send_as_json=False)
regressor.fit(input_data,
output_data,
epochs=1000,
shuffle=True,
callbacks=[rlr, mcp, tb, remote],
validation_split=0.2,
verbose=1,
batch_size=64)
model_json = regressor.to_json()
with open('lstm_weights/last_model.json', 'w') as json_file:
json_file.write(model_json)
def create_callbacks(
identifier='',
names=['checkpointer', 'checkpointer_best', 'csv_logger',
'tb_logger']):
""" Create Callbacks for logging during training """
# instantiate list of callbacks
callbacks = []
# handle identifier string
if identifier is not '':
identifier = identifier + '_'
# add different callbacks if they are specified
if 'checkpointer' in names:
# save model weights after each epoch
checkpointer = ModelCheckpoint(filepath=cfg_path['models'] +
identifier +
"model_{epoch:02d}_{val_loss:.2f}.hdf5",
verbose=0,
save_best_only=False)
callbacks.append(checkpointer)
if 'checkpointer_best' in names:
# save best model with lowest test error
checkpointer = ModelCheckpoint(filepath=cfg_path['models'] +
identifier + "model_best.hdf5",
verbose=0,
save_best_only=True)
callbacks.append(checkpointer)
if 'csv_logger' in names:
# log loss and accuracy to csv
csv_logger = CSVLogger(cfg_path['logs'] + identifier + 'training.log')
callbacks.append(csv_logger)
if 'tb_logger' in names:
# Tensorboard logger
tb_logger = TensorBoard(
log_dir=cfg_path['logs'],
histogram_freq=0,
# batch_size=int(cfg['batch_size']),
write_graph=True
# write_grads=False, write_images=False,
# embeddings_freq=0,
# embeddings_layer_names=None,
# embeddings_metadata=None
)
callbacks.append(tb_logger)
if 'remote_logger' in names:
# Remote Logger
# get ip
ip_call = run(['curl', 'checkip.amazonaws.com'], stdout=PIPE)
ip = ip_call.stdout.decode("utf-8").strip("\n").replace('.', '-')
ip_ec2 = 'http://ec2-' + ip + '.compute-1.amazonaws.com'
rem_logger = RemoteMonitor(root=ip_ec2 + ':8080',
path='/publish/epoch/end/',
field='data',
headers=None)
# print / log server
logging.info("Initializing Remote logger at: %s" % (ip_ec2 + ':8080'))
print("Initializing Remote logger at: %s" % (ip_ec2 + ':8080'))
callbacks.append(rem_logger)
if 'log_disk' in names:
logging_cb = LoggingCallback(logging=logging)
callbacks.append(logging_cb)
if 'early_stopping' in names:
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=5,
verbose=1,
mode='auto')
callbacks.append(early_stopping)
if 'reduce_lr_on_plateau' in names:
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=3,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0.0001)
callbacks.append(reduce_lr)
if 'ss_learning_rate' in names:
# learning rate adjustment scheme according to epoch number
def lrnrate_dec(epoch):
if epoch < 18:
res = 0.01
elif epoch < 29:
res = 0.005
elif epoch < 43:
res = 0.001
elif epoch < 52:
res = 5e-4
else:
res = 1e-4
logging.info("Setting learning rate to: %s" % res)
return res
# learning rate decay rule
learning_rate_decay = LearningRateScheduler(lrnrate_dec)
callbacks.append(learning_rate_decay)
if 'ss_decay' in names:
# learning rate adjustment scheme according to epoch number
def decay_dec(epoch):
if epoch < 18:
res = 5e-4
elif epoch < 29:
res = 5e-4
elif epoch < 43:
res = 0
elif epoch < 52:
res = 0
else:
res = 0
logging.info("Setting learning rate decay to: %s" % res)
return res
# learning rate decay rule
learning_rate_decay = LRDecayScheduler(decay_dec)
callbacks.append(learning_rate_decay)
return callbacks
def cfu_training(train_generator, train_labels, eval_generator, output_file):
"""Cyclic freeze/unfreeze head and top convolutional base.
[conv_base] ==> [conv_top] ==> [dense_head]
"""
callbacks = [
ModelCheckpoint(filepath=output_file, verbose=1, save_best_only=True),
EarlyStopping(monitor="val_acc", min_delta=0.0001, patience=5),
ReduceLROnPlateau(monitor="val_loss",
factor=0.2,
verbose=1,
patience=5,
min_lr=0.0001),
RemoteMonitor(root='http://localhost:9000'),
CSVLogger("./logs/last_run.log"),
]
n_steps = len(train_labels) / config.batch_size
fu_steps = 1 # number of freeze-unfreeze cycles
learner = pretrained.ConvLearner("inceptionv3", "adam", config.image_shape)
for idx in range(fu_steps):
# (a) train classifier on-top
print("[Cycle {}]".format(idx))
learner.freeze("conv_top")
learner.unfreeze("head")
learner.recompile()
print(learner.model.summary())
history = learner.model.fit_generator(
train_generator,
steps_per_epoch=n_steps,
epochs=1,
validation_data=eval_generator, # or eval_split
validation_steps=10,
callbacks=callbacks,
verbose=1)
# (2) fine-tune middle convolutional layers
learner.freeze("head")
learner.unfreeze("conv_top")
learner.optimizer = optimizers.RMSprop(lr=1e-5)
learner.recompile()
print(learner.model.summary())
#print("trainable weights", len(model.trainable_weights))
# NOTE: last conv layers of the conv_base require more gentle optimization typically
#model.optimizer =
history = learner.model.fit_generator(
train_generator,
steps_per_epoch=n_steps,
epochs=1,
validation_data=eval_generator, # or eval_split
validation_steps=10,
callbacks=callbacks,
verbose=1)
print("[Retrain head...]")
# retrain head and top convolutional layers
learner.unfreeze("head")
learner.recompile()
history = learner.model.fit_generator(train_generator,
steps_per_epoch=n_steps,
epochs=config.epochs,
validation_data=eval_generator,
validation_steps=10,
callbacks=callbacks,
verbose=1)
return learner.model