def test_cache():
from keras.layers.core import Dense
from theano import pp, function
from theano import config
import cPickle as pkl
# Theano configuration
config.optimizer = 'fast_run'
X = T.matrix()
d = Dense(200, input_dim=1000)
# d1 = Dense(200, input_dim=1000)
d.build()
Y = d(X) + d(X)
z = d(X)
Y1 = z + z
f = function([X], Y)
f1 = function([X], Y1)
# print pp(Y)
# print pp(f.maker.fgraph.outputs[0])
print theano.printing.debugprint(f)
print
print theano.printing.debugprint(f1)
print
print theano.printing.debugprint(z)
pkl.dump(f, open('test.pkl', 'wb'))
pkl.dump(f1, open('test1.pkl', 'wb'))
def build(self):
# list of embedding layers
self.question = []
self.facts = []
self.memory = []
self.Ws = []
self.trainable_weights = []
for i in range(self.hops):
q = BagEmbedding(self.input_dim, self.q_nb_words, self.output_dim,
1, bow_mode=self.bow_mode,
mask_zero=self.mask_zero, dropout=self.dropout)
q.build()
f = BagEmbedding(self.input_dim, self.f_nb_words, self.output_dim,
self.input_length, bow_mode=self.bow_mode,
mask_zero=self.mask_zero, dropout=self.dropout)
f.build()
m = BagEmbedding(self.input_dim, self.f_nb_words, self.output_dim,
self.input_length, bow_mode=self.bow_mode,
mask_zero=self.mask_zero, dropout=self.dropout)
m.build()
self.question.append(q)
self.facts.append(f)
self.memory.append(m)
if i == self.hops-1:
w = Dense(self.output_dim, input_dim=self.output_dim,
activation=self.activation)
else:
w = Dense(self.output_dim, input_dim=self.output_dim,
activation=self.inner_activation)
w.build()
self.Ws.append(w)
for l in (q, f, m, w):
self.trainable_weights += l.trainable_weights
def build(self):
self.lstms = []
for i in range(self.depth):
if i == 0:
self.lstms.append(LSTM(self.output_dim, self.init, self.inner_init,
self.forget_bias_init, self.activation,
self.inner_activation, **self._kwargs))
else:
self._kwargs['input_dim'] = self.output_dim
self.lstms.append(LSTM(self.output_dim, self.init, self.inner_init,
self.forget_bias_init, self.activation,
self.inner_activation, **self._kwargs))
[lstm.build() for lstm in self.lstms]
# Get a flat list of trainable_weights
self.trainable_weights = [weights for lstm in self.lstms for weights in
lstm.trainable_weights]
if self.readout:
self.readout_layer = Dense(self.readout, input_dim=self.output_dim,
activation='softmax')
self.readout_layer.build()
self.trainable_weights.extend(self.readout_layer.trainable_weights)
if self.initial_weights is not None:
self.set_weights(self.initial_weights)
del self.initial_weights
if self.stateful is not None:
self.states = []
for lstm in self.lstms:
self.states.extend(lstm.states)
else:
self.states = [None, None] * self.depth
def test_value():
from keras.layers.core import Dense
from theano import pp, function
theano.config.compute_test_value = 'warn'
# since the test input value is not aligned with the requirement in Dense,
# it will report error quickly. Change 100 to 1000 will be fine.
t_value = np.zeros((500, 1000), dtype=np.float32)
X = T.matrix()
X.tag.test_value = t_value
d = Dense(200, input_dim=1000)
# d1 = Dense(200, input_dim=1000)
d.build()
z = d(X)
f = function([X], z)
# turn it off after
theano.config.compute_test_value = 'off'
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = Sequential()
model.add(Dense(10, input_shape=(x_train.shape[1],)))
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
assert history.history['acc'][-1] >= target
config = optimizers.serialize(optimizer)
optim = optimizers.deserialize(config)
new_config = optimizers.serialize(optim)
new_config['class_name'] = new_config['class_name'].lower()
assert config == new_config
# Test constraints.
model = Sequential()
dense = Dense(10,
input_shape=(x_train.shape[1],),
kernel_constraint=lambda x: 0. * x + 1.,
bias_constraint=lambda x: 0. * x + 2.,)
model.add(dense)
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.train_on_batch(x_train[:10], y_train[:10])
kernel, bias = dense.get_weights()
assert_allclose(kernel, 1.)
assert_allclose(bias, 2.)
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(
64,
(4, 4),
padding='same',
kernel_regularizer=regularizers.l2(reg),
))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(num_classes, activation='softmax', name='act_output'))
print(model.summary())
opt = Nadam(lr=0.0002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004,
clipvalue=3)
model.compile(loss={'act_output': 'categorical_crossentropy'},
optimizer=opt,
metrics=['accuracy'])
early_stopping = EarlyStopping(monitor='val_loss', patience=6)
history = model.fit(X_train, {'act_output': train_Y_one_hot},
validation_split=0.2,
import os
#Loading our single test image: This can be easily adapted to work on the video frames from the robot
img_path = "/home/bxv7657/but/0030030.png"
im2 = cv2.resize(cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB), (224, 224))
im = np.expand_dims(im2, axis=0)
#Using a VGG16 pre-trained on ImageNet
resmo = VGG16(weights=None, include_top=True)
#Freezing all the above layers
for layer in (resmo.layers):
layer.trainable = False
#Adding a new Dense layer with 10 output nodes for 10 species of butterflies
x = Dense(10, activation='softmax', name='predictions')(resmo.layers[-2].output)
#Redeffining a new model with 10 output class nodes
my_model = Model(inputs=resmo.input,outputs=(x))
my_model.compile(optimizer="sgd", loss='categorical_crossentropy',metrics=['accuracy'])
#Loading weight file generated after running train.py
my_model.load_weights('butterflyvgg_weights.h5')
my_model.compile(optimizer="sgd", loss='categorical_crossentropy',metrics=['accuracy'])
preds=my_model.predict(im)
#Printing the name of species from the label.txt file
y_classes = np.argmax(preds)
with open('label.txt') as class_file:
class_dict = ast.literal_eval(class_file.read())
fig, ax = plt.subplots()
def alexnet_model(img_shape=(50, 50, 3), n_classes=2, l2_reg=0.,
weights=None):
# Initialize model
alexnet = Sequential()
# Layer 1
alexnet.add(Conv2D(96, (11, 11), input_shape=img_shape,
padding='same', kernel_regularizer=l2(l2_reg)))
alexnet.add(BatchNormalization())
alexnet.add(Activation('relu'))
alexnet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 2
alexnet.add(Conv2D(256, (5, 5), padding='same'))
alexnet.add(BatchNormalization())
alexnet.add(Activation('relu'))
alexnet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 3
alexnet.add(ZeroPadding2D((1, 1)))
alexnet.add(Conv2D(512, (3, 3), padding='same'))
alexnet.add(BatchNormalization())
alexnet.add(Activation('relu'))
alexnet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 4
alexnet.add(ZeroPadding2D((1, 1)))
alexnet.add(Conv2D(1024, (3, 3), padding='same'))
alexnet.add(BatchNormalization())
alexnet.add(Activation('relu'))
# Layer 5
alexnet.add(ZeroPadding2D((1, 1)))
alexnet.add(Conv2D(1024, (3, 3), padding='same'))
alexnet.add(BatchNormalization())
alexnet.add(Activation('relu'))
alexnet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 6
alexnet.add(Flatten())
alexnet.add(Dense(3072))
alexnet.add(BatchNormalization())
alexnet.add(Activation('relu'))
alexnet.add(Dropout(0.5))
# Layer 7
alexnet.add(Dense(4096))
alexnet.add(BatchNormalization())
alexnet.add(Activation('relu'))
alexnet.add(Dropout(0.5))
# Layer 8
alexnet.add(Dense(n_classes))
alexnet.add(BatchNormalization())
alexnet.add(Activation('softmax'))
if weights is not None:
alexnet.load_weights(weights)
return alexnet
def pretrain(self, batch_size=200, act='relu', nEpoch = 200):
# we have to instantiate the model over and over again because we cannot freeze a layer after its been compiled
self.weights = []
for cSpecs in enumerate(zip(self.modelStructure, self.dropOuts), start=1):
# cSpecs is e.g. (1, (500, 0.8)) where the first element gives how many hidden layers we have right now
nLayers = cSpecs[0]
nOut = cSpecs[1][0]
pDrop = cSpecs[1][1]
if nLayers == 1:
# we have input -> hidden -> output
model = Sequential()
hiddenLayer = Dense(nOut, activation=act, input_dim=self.nFeatures) # first hidden layer
dropoutLayer = Dropout(pDrop) # dropout layer
outputLayer = Dense(1) # output layer (we have only one if nLayers == 0)
model.add(hiddenLayer)
model.add(dropoutLayer)
model.add(outputLayer)
# fit model
adam = Adam(lr=0.00001)
model.compile(loss='mse', optimizer=adam)
model.fit(self.X_train, self.y_train, batch_size=batch_size, validation_split=0.2,
show_accuracy=True, verbose=1, nb_epoch=nEpoch)
# save weights of hiddenLayer
self.weights.append(hiddenLayer.get_weights())
# instantiate a fresh model after grabbing the weights from the first hidden layer
model = Sequential()
else:
# we have input -> hidden -> .. -> hidden -> output (at least two hidden layers)
# nLayer indicates how many hidden layers we have (counting from 0)
# first build frozen layers up to the current one
for cLayer in range(nLayers):
nOut_cc = self.modelStructure[cLayer]
pDrop_cc = self.dropOuts[cLayer]
print "nlayers = ", nLayers
print "clayer = ", cLayer
print "layers.shape = ", len(self.weights)
if cLayer == 0:
# build first frozen layer
model.add(Dense(nOut_cc, weights=self.weights[cLayer], input_dim=self.nFeatures, activation=act, trainable=False))
model.add(Dropout(pDrop_cc))
elif cLayer < nLayers-1:
# build next frozen layer
model.add(Dense(nOut_cc, weights=self.weights[cLayer], activation=act, trainable=False))
model.add(Dropout(pDrop_cc))
else:
# build new hidden layer (which we want to train)
# Caveat: Now we have to use nOut # of hidden units and pDrop as dropout parameter
# as this is the hidden layer we add to the network
hiddenLayer = Dense(nOut, activation=act)
dropoutLayer = Dropout(pDrop)
outputLayer = Dense(1)
model.add(hiddenLayer)
model.add(dropoutLayer)
model.add(outputLayer)
# fit model
adam = Adam(lr=0.00001)
model.compile(loss='mse', optimizer=adam)
model.fit(self.X_train, self.y_train, batch_size=batch_size, validation_split=0.2,
show_accuracy=True, verbose=1, nb_epoch=nEpoch)
# save weights of hiddenLayer
self.weights.append(hiddenLayer.get_weights())
model = Sequential()
print "Pretraining complete"
def simple_gan_generator(nb_units, z, labels, depth_map, tag3d, depth=2):
n = nb_units
depth_map_features = sequential([
conv2d_block(n),
conv2d_block(2 * n),
])(depth_map)
tag3d_features = sequential([
conv2d_block(n, subsample=2),
conv2d_block(2 * n, subsample=2),
])(tag3d)
x = sequential([
Dense(5 * n),
BatchNormalization(mode=2),
Activation('relu'),
Dense(5 * n),
BatchNormalization(mode=2),
Activation('relu'),
])(concat([z, labels]))
blur = InBounds(0, 1, clip=True)(Dense(1)(x))
x = sequential([
Dense(8 * 4 * 4 * n),
Activation('relu'),
BatchNormalization(mode=2),
Reshape((8 * n, 4, 4)),
])(x)
x = sequential([
conv2d_block(8 * n, filters=1, depth=1, up=True), # 4x4 -> 8x8
conv2d_block(8 * n, depth=depth, up=True), # 8x8 -> 16x16
])(x)
off_depth_map = sequential([
conv2d_block(2 * n, depth=depth),
])(concat([x, depth_map_features]))
light = sequential([
conv2d_block(2 * n, depth=depth, up=True), # 16x16 -> 32x32
conv2d_block(n, depth=depth, up=True), # 32x32 -> 64x64
])(off_depth_map)
def get_light(x):
return sequential([
conv2d_block(1, filters=1, batchnorm=False),
GaussianBlur(sigma=4),
InBounds(0, 1, clip=True),
])(x)
light_sb = get_light(light)
light_sw = get_light(light)
light_t = get_light(light)
background = sequential([
conv2d_block(2 * n, depth=depth, up=True), # 16x16 -> 32x32
conv2d_block(n, depth=depth, up=True), # 32x32 -> 64x64
conv2d_block(1, batchnorm=False),
InBounds(-1, 1, clip=True),
])(off_depth_map)
details = sequential([
conv2d_block(2 * n, depth=depth, up=True), # 16x16 -> 32x32
conv2d_block(n, depth=depth, up=True), # 32x32 -> 64x64
conv2d_block(1, depth=1, batchnorm=False),
InBounds(-1, 1, clip=True)
])(concat(tag3d_features, off_depth_map))
return blur, [light_sb, light_sw, light_t], background, details
conv_layer1 = lstm(conv_layer1)
layers.append(conv_layer1)
encoder_size += rnn_size
classifier = keras.layers.concatenate(inputs=list(layers))
# --------------------------------------------------------------------------
# финальный классификатор определяет способ получения ответа:
# 1) да/нет
# 2) ответ строится копированием слов вопроса
# 3) текст ответа генерируется сеткой
# 4) ответ посимвольно генерируется сеткой и содержит одни цифры
output_dims = 4
classifier = Dense(encoder_size, activation='sigmoid')(classifier)
#classifier = Dense(encoder_size//2, activation='relu')(classifier)
#classifier = Dense(encoder_size//3, activation='relu')(classifier)
classifier = Dense(output_dims, activation='softmax',
name='output')(classifier)
model = Model(inputs=inputs, outputs=classifier)
model.compile(loss='categorical_crossentropy',
optimizer='nadam',
metrics=['accuracy'])
model.summary()
with open(arch_filepath, 'w') as f:
f.write(model.to_json())
# -------------------------------------------------------------------------
def faceRecoModel(input_shape):
"""
Implementation of the Inception model used for FaceNet
Arguments:
input_shape -- shape of the images of the dataset
Returns:
model -- a Model() instance in Keras
"""
# Define the input as a tensor with shape input_shape
X_input = Input(input_shape)
# Zero-Padding
X = ZeroPadding2D((3, 3))(X_input)
# First Block
X = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(X)
X = BatchNormalization(axis=1, name='bn1')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
X = MaxPooling2D((3, 3), strides=2)(X)
# Second Block
X = Conv2D(64, (1, 1), strides=(1, 1), name='conv2')(X)
X = BatchNormalization(axis=1, epsilon=0.00001, name='bn2')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
# Second Block
X = Conv2D(192, (3, 3), strides=(1, 1), name='conv3')(X)
X = BatchNormalization(axis=1, epsilon=0.00001, name='bn3')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
X = MaxPooling2D(pool_size=3, strides=2)(X)
# Inception 1: a/b/c
X = inception_block_1a(X)
X = inception_block_1b(X)
X = inception_block_1c(X)
# Inception 2: a/b
X = inception_block_2a(X)
X = inception_block_2b(X)
# Inception 3: a/b
X = inception_block_3a(X)
X = inception_block_3b(X)
# Top layer
X = AveragePooling2D(pool_size=(3, 3),
strides=(1, 1),
data_format='channels_first')(X)
X = Flatten()(X)
X = Dense(128, name='dense_layer')(X)
# L2 normalization
X = Lambda(lambda x: K.l2_normalize(x, axis=1))(X)
# Create model instance
model = Model(inputs=X_input, outputs=X, name='FaceRecoModel')
return model
def train():
model = Sequential()
X_train = np.load(home + '/gabor/numpyFiles/Training Set.npy')
X_test = np.load(home + '/gabor/numpyFiles/TestSet.npy')
Y_train = np.load(home + '/gabor/numpyFiles/Training Labels.npy')
Y_test = np.load(home + '/gabor/numpyFiles/TestSet Labels.npy')
#X_test = X_test.reshape(-1, 1, 30, 96)
Y_test = np_utils.to_categorical(Y_test, 447)
#X_train = X_train.reshape(-1, 1, 30, 96)
Y_train = np_utils.to_categorical(Y_train, 447)
print("X_test.shape == {};".format(X_test.shape))
print("Y_test.shape == {};".format(Y_test.shape))
print("X_test.shape == {};".format(X_train.shape))
print("Y_test.shape == {};".format(Y_train.shape))
nb_hidden_layers = [len(X_train[0]), 700, 500, 300]
XtrainKeras = X_train
print 'shape of XTrain Keras is ', XtrainKeras.shape
YtrainKeras = np_utils.to_categorical(Y_train, nb_classes)
op1 = RMSprop(lr=0.01, rho=0.5, epsilon=1e-8)
X_train_tmp = XtrainKeras
trained_encoders = []
#XtrainKeras=XwhiteTrain.reshape(-1,1,len(XwhiteTrain),len(XwhiteTrain[0]))
#YtrainKeras=np_utils.to_categorical(Y_train, nb_classes)
#XtestKeras=X_test.reshape(-1,1,imageWidth,imageHeight)
#YtestKeras=np_utils.to_categorical(Y_test, nb_classes)
#X_train_tmp=XtrainKeras
for n_in, n_out in zip(nb_hidden_layers[:-1], nb_hidden_layers[1:]):
print('Pre-training the layer: Input {} -> Output {}'.format(
n_in, n_out))
# Create AE and training
ae = Sequential()
encoder = containers.Sequential(
[Dense(n_out, input_dim=n_in, activation='sigmoid')])
decoder = containers.Sequential(
[Dense(n_in, input_dim=n_out, activation='sigmoid')])
ae.add(
AutoEncoder(encoder=encoder,
decoder=decoder,
output_reconstruction=False))
ae.compile(loss='mean_squared_error', optimizer=op1)
hist = ae.fit(X_train_tmp,
X_train_tmp,
batch_size=batch_size,
nb_epoch=nb_epoch)
print(hist.history)
Fname = prefix + 'autoencoder_n_in=' + str(n_in) + '_n_out= ' + str(
n_out) + '.json'
weightName = prefix + 'Weights_autoencoder_n_in=' + str(
n_in) + '_n_out= ' + str(n_out) + '.h5'
json_string = model.to_json()
open(Fname, 'w').write(json_string)
model.save_weights(weightName, overwrite=True)
# Store trainined weight
trained_encoders.append(ae.layers[0].encoder)
# Update training data
X_train_tmp = ae.predict(X_train_tmp)
#ae1=Sequential()
#encoder1=containers.Sequential([Dense(len(XwhiteTrain[0])-200,len(XwhiteTrain[0]),activation='sigmoid')])
Y_test = np_utils.to_categorical(Y_test, nb_classes)
#X_test=X_test.reshape(-1,len(X_test[0]))
print 'shape of X_test is ', X_test.shape
print('Fine-tuning')
sgd = SGD(lr=0.01, momentum=0.5, decay=0., nesterov=False)
i = 1
model = Sequential()
for encoder in trained_encoders:
model.add(encoder)
model.add(
Dense(nb_classes, input_dim=nb_hidden_layers[-1],
activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=sgd)
hist = model.fit(XtrainKeras,
YtrainKeras,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
validation_data=(X_test, Y_test))
score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
Fname = prefix + '2 FineTuning_model=' + '.json'
weightName = prefix + 'Fine Tunes Weights_autoencoder_i=' + str(i) + '.h5'
json_string = model.to_json()
open(Fname, 'w').write(json_string)
model.save_weights(weightName, overwrite=True)
model.add(Dropout(0.25))
model.add(
Convolution2D(nb_filters * 2,
nb_conv,
nb_conv,
border_mode='valid',
input_shape=(1, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(nb_filters * 2, nb_conv, nb_conv))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
hist = model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
verbose=1,
validation_data=(X_valid, Y_valid))
Train_Result_Optimizer = hist.history
Train_Loss = np.asarray(Train_Result_Optimizer.get('loss'))
def Autoencoder(x_train, y_train, x_test, y_test):
input_shape = (x_train.shape[1],)
input2 = Input(input_shape)
encoded = Dense(80, activation='relu',
kernel_initializer='glorot_uniform',
name='encod0')(input2)
encoded = Dense(30, activation='relu',
kernel_initializer='glorot_uniform',
name='encod1')(encoded)
encoded = Dense(10, activation='relu',
kernel_initializer='glorot_uniform',
name='encod2')(encoded)
encoded= Dropout({{uniform(0, 1)}})(encoded)
decoded = Dense(30, activation='relu',
kernel_initializer='glorot_uniform',
name='decoder1')(encoded)
decoded = Dense(80, activation='relu',
kernel_initializer='glorot_uniform',
name='decoder2')(decoded)
decoded = Dense(x_train.shape[1], activation='linear',
kernel_initializer='glorot_uniform',
name='decoder3')(decoded)
model = Model(inputs=input2, outputs=decoded)
model.summary()
adam=Adam(lr={{uniform(0.0001, 0.01)}})
model.compile(loss='mse', metrics=['acc'],
optimizer=adam)
callbacks_list = [
callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=10,
restore_best_weights=True),
]
XTraining, XValidation, YTraining, YValidation = train_test_split(x_train, x_train, stratify=y_train,
test_size=0.2) # before model building
tic = time.time()
history= model.fit(XTraining, YTraining,
batch_size={{choice([32,64, 128,256,512])}},
epochs=150,
verbose=2,
callbacks=callbacks_list,
validation_data=(XValidation,YValidation))
toc = time.time()
score = np.amin(history.history['val_loss'])
print('Best validation loss of epoch:', score)
scores = [history.history['val_loss'][epoch] for epoch in range(len(history.history['loss']))]
score = min(scores)
print('Score',score)
print('Best score',global_config.best_score)
if global_config.best_score > score:
global_config.best_score = score
global_config.best_model = model
global_config.best_numparameters = model.count_params()
global_config.best_time = toc - tic
return {'loss': score, 'status': STATUS_OK, 'n_epochs': len(history.history['loss']), 'n_params': model.count_params(), 'model': global_config.best_model, 'time':toc - tic}
X_test = np.arange(0.005, 1.005, 0.01)
y_test = [0 for ii in range(len(X_test))]
for ii in range(0, len(X_test)):
if np.logical_and(X_test[ii] > 0.2, X_test[ii] < 0.6):
y_test[ii] = (0, 1)
else:
y_test[ii] = (1, 0)
X_train = X_train.reshape(X_train.shape[0], 1) #had tough time without this,
# y_train=y_train.reshape(y_train.shape[0],1) #had tough time without this,
X_test = X_test.reshape(X_test.shape[0], 1) #had tough time without this,
# y_test=y_test.reshape(y_test.shape[0],1) #had tough time without this,
model = Sequential()
model.add(Dense(1, 500, init='normal', activation='tanh'))
model.add(Dense(500, 500, init='normal', activation='tanh'))
# model.add(Dropout(0.5))
model.add(Dense(500, 500, init='uniform', activation='tanh'))
# model.add(Dropout(0.5))
# model.add(Dense(50, 50, init='uniform', activation='tanh'))
# model.add(Dropout(0.5))
model.add(Dense(500, 2, init='normal', activation='softmax'))
# model.add(Activation('linear'))
sgd = SGD(lr=0.05, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error', optimizer=sgd)
model.fit(X_train, y_train, nb_epoch=5000, batch_size=use_batch_size)
def build(width, height, depth, classes):
# initialize the model along with the input shape to be
# "channels last" and the channels dimension itself
model = Sequential()
inputShape = (height, width, depth)
chanDim = -1
# if we are using "channels first", update the input shape
# and channels dimension
if K.image_data_format() == "channels_first":
inputShape = (depth, height, width)
chanDim = 1
# CONV => RELU => POOL
model.add(Conv2D(64, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(Conv2D(64, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(128, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(Conv2D(128, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(256, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(Conv2D(256, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(Conv2D(256, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(512, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(Conv2D(512, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(Conv2D(512, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(512, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(Conv2D(512, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(Conv2D(512, (3, 3), padding="same", input_shape=inputShape))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# first (and only) set of FC => RELU layers
model.add(Flatten())
model.add(Dense(4096))
model.add(Activation("relu"))
model.add(Dropout(0.5))
model.add(Dense(4096)
model.add(Activation("relu"))
model.add(Dropout(0.5))
# softmax classifier
model.add(Dense(classes))
model.add(Activation("softmax"))
# return the constructed network architecture
return model
execfile('00_readingInput.py')
''' Import l1,l2 (regularizer) '''
from keras.regularizers import l1,l2
''' set the size of mini-batch and number of epochs'''
batch_size = 16
epochs = 50
''' Import keras to build a DL model '''
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
print('Building a model with regularizer L2')
model_l2 = Sequential()
model_l2.add(Dense(128, input_dim=200, kernel_regularizer=l2(0.005)))
model_l2.add(Activation('relu'))
model_l2.add(Dense(256, kernel_regularizer=l2(0.005)))
model_l2.add(Activation('relu'))
model_l2.add(Dense(5, kernel_regularizer=l2(0.005)))
model_l2.add(Activation('softmax'))
''' Setting optimizer as Adam '''
from keras.optimizers import SGD, Adam, RMSprop, Adagrad
model_l2.compile(loss= 'categorical_crossentropy',
optimizer='Adam',
metrics=['accuracy'])
'''Fit models and use validation_split=0.1 '''
history_l2 = model_l2.fit(X_train, Y_train,
batch_size=batch_size,
"""
Gets the unique answers and store them in a .sav file
"""
lbl = LabelEncoder()
lbl.fit(answers_train)
nb_classes = len(list(lbl.classes_))
pk.dump(lbl, open('v1/label_encoder_mlp.sav', 'wb'))
"""
Building the MLP VQA model and compiling the model
"""
model = Sequential(name="MLP model")
model.add(
Dense(num_hidden_units,
input_dim=word2vec_dim + img_dim,
kernel_initializer='uniform',
name="feeding_comined_image_question_vector"))
model.add(Dropout(dropout, name="Dropout_1_0.5"))
for i in range(num_hidden_layers):
name_d = "MLP_" + str(i + 1) + "_Hidden_layer_size_1000"
model.add(
Dense(num_hidden_units, kernel_initializer='uniform', name=name_d))
name_a = "Activation_" + str(i + 1) + "_tanh"
model.add(Activation(activation, name=name_a))
temp = "Dropout_" + str(i + 2) + "_0.5"
model.add(Dropout(dropout, name=temp))
model.add(
Dense(nb_classes,
kernel_initializer='uniform',
name="MLP_output_layer_size_1000"))
model.add(Activation('softmax', name="softmax_output_Probabilities"))
feature = ['F1', 'F2', 'F3', 'F4'] #影响因素四个
label = ['L1'] #标签一个，即需要进行预测的值
data_train = data.loc[range(0, 6)].copy() #标明excel表从第0行到520行是训练集
#2 数据预处理和标注
data_mean = data_train.mean()
data_std = data_train.std()
data_train = (data_train - data_mean) / data_std #数据标准化
x_train = data_train[feature].as_matrix() #特征数据
y_train = data_train[label].as_matrix() #标签数据
#3 建立一个简单BP神经网络模型
from keras.models import Sequential
from keras.layers.core import Dense, Activation
model = Sequential() #层次模型
model.add(Dense(12, input_dim=4, init='uniform')) #输入层，Dense表示BP层
model.add(Activation('relu')) #添加激活函数
model.add(Dense(1, input_dim=12)) #输出层
model.compile(loss='mean_squared_error', optimizer='adma') #编译模型
model.fit(x_train, y_train, nb_epoch=1000, batch_size=6) #训练模型1000次
model.save_weights(modelfile) #保存模型权重
#4 预测，并还原结果。
x = ((data[feature] - data_mean[feature]) / data_std[feature]).as_matrix()
data[u'L1_pred'] = model.predict(x) * data_std['L1'] + data_mean['L1']
#5 导出结果
data.to_excel(outputfile)
#6 画出预测结果图
import matplotlib.pyplot as plt
with open('bfvy_resnet.pkl', 'rb') as f:
y_val2 = pickle.load(f)
# X_train2 = datax['features']
# y_train2 = datax['labels']
# X_val2 = datay['features']
# y_val2 = datay['labels']
return X_train2, y_train2, X_val2, y_val2
X_train, y_train, X_val, y_val = load_bottleneck_data('resnet_train_bottleneck.json',
'resnet_validate_bottleneck.json')
input_shape = X_train.shape[1:]
inp = Input(shape=input_shape)
x = Flatten()(inp)
x = Dense(num_classes, activation='softmax')(x)
model = Model(inp, x)
sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy',f1])
with tf.Session() as sess:
# fetch session so Keras API can work
K.set_session(sess)
K.set_learning_phase(1)
history =model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,
validation_data=(X_val, y_val), shuffle=True, verbose=1 )
model.save_weights('resnet_bottleneck_weights.h5')
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
print(df.head())
'''
dataset = df.values
X = dataset[:, 0:60]
Y_obj = dataset[:, 60]
e = LabelEncoder()
e.fit(Y_obj)
Y = e.transform(Y_obj)
# 학습 셋과 테스트 셋의 구분
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
test_size=0.3,
random_state=seed)
model = Sequential()
model.add(Dense(24, input_dim=60, activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='mean_squared_error',
optimizer='adam',
metrics=['accuracy'])
model.fit(X_train, Y_train, epochs=130, batch_size=5)
# 테스트셋에 모델 적용
print("\n Test Accuracy: %.4f" % (model.evaluate(X_test, Y_test)[1]))
def get_dense(nb):
return [
Dense(nb),
batch_norm(),
LeakyReLU(0.2),
]
def run(df,fname):
# parameters
epsilon_0 = .001
num_actions = 3
epoch = 5000
max_memory = 100
batch_size = 500
lkbk = 3
START_IDX = 300
env = Game(df, lkbk=lkbk, max_game_len=1000,init_idx=START_IDX,run_mode='sequential')
hidden_size = num_actions*len(env.state)*2
model = Sequential()
model.add(Dense(hidden_size, input_shape=(len(env.state),), activation='relu'))
model.add(Dense(hidden_size, activation='relu'))
model.add(Dense(num_actions))
model.compile(SGD(lr=.05), "mse")
# If you want to continue training from a previous model, just uncomment the line bellow
# model.load_weights("indicator_model.h5")
# Initialize experience replay object
exp_replay = ExperienceReplay(max_memory=max_memory)
# Train
win_cnt = 0
loss_cnt = 0
wins = []
losses = []
pnls = []
for e in range(epoch):
action = 0
# epsilon = epsilon_0**(np.log10(e))
epsilon = 0.4
env = Game(df, lkbk=lkbk, max_game_len=1000,init_idx=env.curr_idx,run_mode='sequential')
loss = 0.
env.reset()
game_over = False
# get initial input
input_t = env.observe()
cnt = 0
while not game_over:
print(cnt)
cnt += 1
input_tm1 = input_t
# get next action
if env.position:
print('***Time Exit***')
action = exit_action
elif np.random.rand() <= epsilon:
action = np.random.randint(0, num_actions, size=1)[0]
if env.position == 0:
if action != 0:
print('***random entry***')
input_state_start = deepcopy(input_tm1)
action_start = action
if action == 2:
exit_action = 1
elif action == 1:
exit_action = 2
elif env.position == 0:
q = model.predict(input_tm1)
action = np.argmax(q[0])
if action:
input_state_start = deepcopy(input_tm1)
action_start = action
exit_action = np.argmin(q[0][1:])+1
# apply action, get rewards and new state
input_t, reward, game_over = env.act(action)
if reward > 0:
win_cnt += 1
elif reward < 0:
loss_cnt += 1
# store experience
# if action or len(exp_replay.memory)<20 or np.random.rand() < 0.1:
# exp_replay.remember([input_tm1, action, reward, input_t], game_over)
# inputs, targets = exp_replay.get_batch(model, batch_size=batch_size)
env.pnl_sum = sum(pnls)
# zz = model.train_on_batch(inputs, targets)
# loss += zz
exp_replay.remember(input_state_start, action_start, reward)
inputs, targets = exp_replay.get_batch(model, batch_size=batch_size)
loss = model.train_on_batch(inputs, targets)
prt_str = ("Epoch {:03d} | Loss {:.2f} | pos {} | len {} | sum pnl {:.2f}% @ {:.2f}% | eps {:,.4f} | {} | entry price {} | current price {}".format(e,
loss,
env.position,
env.trade_len,
sum(pnls)*100,
env.pnl*100,
epsilon,
env.curr_time, env.entry, env.curr_price))
print(prt_str)
fid = open(fname,'a')
fid.write(prt_str+'\n')
fid.close()
pnls.append(env.pnl)
if not e%10:
print('----saving weights-----')
model.save_weights("indicator_model.h5", overwrite=True)
pn['sent'] = pn['words'].apply(get_sent) #速度太慢
maxlen = 50
print("Pad sequences (samples x time)")
pn['sent'] = list(sequence.pad_sequences(pn['sent'], maxlen=maxlen))
x = np.array(list(pn['sent']))[::2] #训练集
y = np.array(list(pn['mark']))[::2]
xt = np.array(list(pn['sent']))[1::2] #测试集
yt = np.array(list(pn['mark']))[1::2]
xa = np.array(list(pn['sent'])) #全集
ya = np.array(list(pn['mark']))
print('Build model...')
model = Sequential()
model.add(Embedding(len(dict) + 1, 256))
model.add(LSTM(256, 128)) # try using a GRU instead, for fun
model.add(Dropout(0.5))
model.add(Dense(128, 1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
class_mode="binary")
model.fit(xa, ya, batch_size=16, nb_epoch=10) #训练时间为若干个小时
classes = model.predict_classes(xa)
acc = np_utils.accuracy(classes, ya)
print('Test accuracy:', acc)
#使用one hot encoding 来处理数据
y_train = np_utils.to_categorical(y_train, 10)
y_test = np_utils.to_categorical(y_test, 10)
model = Sequential()
model.add(Convolution2D(25, (3, 3), input_shape=(28, 28, 1)))
model.add(MaxPool2D(2, 2))
model.add(Convolution2D(50, (3, 3)))
model.add(MaxPool2D(2, 2))
model.add(Flatten())
#第一个隐层
model.add(Dense(units=512, kernel_initializer='he_normal', activation='relu'))
model.add(Dropout(0.2)) #dropout 防止过拟合
#第二层
model.add(Dense(units=512, kernel_initializer='he_normal', activation='relu'))
model.add(Dropout(0.2)) #dropout 防止过拟合
model.add(Dense(units=10, activation='softmax')) #输出
#训练
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(x_train,
y_train,
batch_size=64,
epochs=20,
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.optimizers import Adadelta
input_shape = (3, 32, 32)
nb_classes = 10
model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same', input_shape=input_shape))
model.add(Activation('relu'))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=Adadelta(),
metrics=['accuracy'])
numOfPrevSteps = trainX.shape[1]
featurelen = trainX.shape[2]
plt.plot(X_t)
plt.plot(Y_t)
plt.show()
from keras.models import Sequential
from keras.layers.core import Dense, Activation, Dropout
from keras.layers import Dense, LSTM, SimpleRNN, GRU, Dropout
print('Building model...')
model = Sequential()
model.add(LSTM(50, batch_input_shape=trainX.shape, stateful=True))
model.add(Dense(featurelen))
model.add(Activation('softmax'))
model.compile(loss='mean_squared_error', optimizer='adam')
model.reset_states()
print('starting training')
num_epochs = 100
for e in range(num_epochs):
print('epoch - ', e + 1)
for i in range(trainX.shape[0] - 1):
model.train_on_batch(
trainX[i:i + 1, :, :], trainY[i:i + 1, :]
) # Train on guessing a single element based on the previous element
model.reset_states()
for i in range(100):
pred = model.predict(np.array([[[1]]]))
with open(MODEL_LABELS_FILENAME, "wb") as f:
pickle.dump(lb, f)
# Build the neural network!
model = Sequential()
# First convolutional layer with max pooling
model.add(Conv2D(20, (5, 5), padding="same", input_shape=(20, 20, 1), activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Second convolutional layer with max pooling
model.add(Conv2D(50, (5, 5), padding="same", activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Hidden layer with 500 nodes
model.add(Flatten())
model.add(Dense(500, activation="relu"))
# Output layer with 32 nodes (one for each possible letter/number we predict)
model.add(Dense(32, activation="softmax"))
# Ask Keras to build the TensorFlow model behind the scenes
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
# Train the neural network
model.fit(X_train, Y_train, validation_data=(X_test, Y_test), batch_size=32, epochs=1, verbose=1)
# Save the trained model to disk
model.save(MODEL_FILENAME)
open('completed'+str(time.time())+'.txt', 'a').close()
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
# Layer 4 - Convolution-Convolution-Normalizing-MaxPooling - Dropout 0.5
model.add(Conv2D(256, (3,3), padding = "valid", activation = 'relu'))
model.add(BatchNormalization(axis=-1))
model.add(Conv2D(256, (3,3), padding = "valid", activation = 'relu'))
model.add(BatchNormalization(axis=-1))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.5))
# Layer 5 - Flattening
model.add(Flatten())
# Layer 6 - FCC Input Layer - Dropout 0.5
model.add(Dense(1024))
model.add(Activation("relu"))
model.add(BatchNormalization())
model.add(Dropout(0.5))
# Layer 7 - FCC Output Layer - Activation Softmax
model.add(Dense(2))
model.add(Activation("softmax"))
# Compiling
model.compile(optimizer = currOptimizer, loss = "binary_crossentropy", metrics = ["accuracy"])
# Model Training
trainedModel = model.fit_generator(data_augemntation.flow(X_train, Y_train, batch_size = numBatches),
validation_data = (X_test, Y_test),
steps_per_epoch = len(X_train) // numBatches,
def construct_mlp(saved_weights, modelStructure=None, dropOuts=None, nFeatures=nFeat):
# construct MLP and set the layer weights to the saved weights
if modelStructure is None:
modelStructure = [500, 100, 20]
if dropOuts is None:
dropOuts = [0.8, 0.5, 0.5]
model = Sequential()
hiddenLayer1 = Dense(output_dim=500, activation='relu', input_dim=nFeat, trainable=False)
#dropOut1 = Dropout(p=0.8)
model.add(hiddenLayer1)
hiddenLayer1.set_weights(saved_weights[0])
hiddenLayer2 = Dense(output_dim=100, activation='relu', input_dim=500, trainable=False)
#dropOut2 = Dropout(p=0.5)
model.add(hiddenLayer2)
hiddenLayer2.set_weights(saved_weights[1])
hiddenLayer3 = Dense(output_dim=20, activation='relu', input_dim=100, trainable=False)
#dropOut3 = Dropout(p=0.5)
model.add(hiddenLayer3)
hiddenLayer3.set_weights(saved_weights[2])
outputLayer = Dense(1, input_dim=20, trainable=False)
model.add(outputLayer)
outputLayer.set_weights(saved_weights[3])
adam = Adam(lr=0.00001)
model.compile(loss='mse', optimizer=adam)
return model
for word in words:
if word in word2index:
seqs.append(word2index[word])
else:
seqs.append(word2index["UNK"])
X[i] = seqs
y[i] = int(label)
i += 1
ftrain.close()
Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.2, random_state=42)
EMBEDDING_SIZE = 128
HIDDEN_LAYER_SIZE = 64
BATCH_SIZE = 32
NUM_EPOCHS = 10
model = Sequential()
model.add(Embedding(vocab_size, output_dim=EMBEDDING_SIZE, input_length=MAX_SENTENCE_LENGTH))
model.add(SpatialDropout1D(0.2))
model.add(LSTM(HIDDEN_LAYER_SIZE, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1))
model.add(Activation("sigmoid"))
model.summary()
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
history = model.fit(Xtrain, ytrain, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, validation_data=(Xtest, ytest))
score, acc = model.evaluate(Xtest, ytest, batch_size=BATCH_SIZE)
print("Test score: %.3f, accuracy: %.3f" % (score, acc))
import theano
import theano.tensor as T
import matplotlib.pyplot as plt
import pickle
import numpy as np
from keras import backend as K
import datetime
from keras.datasets import mnist
from keras.models import Sequential, Graph
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.utils import np_utils
model = Sequential()
model.add(Convolution2D(200, 2, 2, border_mode='same', input_shape=(1,20,20)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
d = Dense(1500, W_regularizer=l2(1e-3), activation='relu')
model.add(d)
model.add(Dropout(0.5))
model.add(Dense(1))
# model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
c = theano.function([d.get_input(train=False)], d.get_output(train=False))
o = c(np.random.random((1,20000)).astype('float32'))
print(d.input_shape)
# X_train /= 255
# X_test /= 255
model = VGG_16("./weights/vgg16_weights.h5")
# model = convnet('VGG_16',weights_path="./weights/vgg16_weights.h5", heatmap=False)
pop_layer(model)
print len(model.layers)
for layer in model.layers:
layer.trainable = False
layer_last = Dense(13, activation='softmax')
layer_last.trainable = True
model.add(layer_last)
sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd, loss='mse', metrics=['accuracy'])
if not os.path.exists("./temporal_weights"):
os.makedirs("./temporal_weights")
checkpointer = ModelCheckpoint(filepath="./temporal_weights/weights_finetunned_vgg_terrassa.hdf5", verbose=1,
save_best_only=True)
if not data_augmentation:
print('Not using data augmentation.')
def __init__(self, output_dim, hidden_dim,output_length, init='glorot_uniform', inner_init='orthogonal', forget_bias_init='one', activation='tanh', inner_activation='hard_sigmoid',
weights=None, truncate_gradient=-1,
input_dim=None, input_length=None, hidden_state=None, batch_size=None, depth=2, context_sensitive=False,
):
if not type(depth) == list:
depth = [depth, depth]
n_lstms = sum(depth)
if depth[1] < 2 and context_sensitive:
print "Warning: Your model will not be able to remember its previous output!"
if weights is None:
weights = [None] * (n_lstms + 1)
if hidden_state is None:
hidden_state = [None] * (n_lstms + 1)
encoder_index = depth[0] - 1
decoder_index = depth[0] + 1
decoder = LSTMDecoder2(dim=output_dim, hidden_dim=hidden_dim, output_length=output_length,
init=init,inner_init=inner_init, activation=activation,
inner_activation=inner_activation,weights=weights[decoder_index],
truncate_gradient = truncate_gradient,
hidden_state=hidden_state[decoder_index], batch_size=batch_size, remember_state=context_sensitive)
encoder = LSTMEncoder(input_dim=input_dim, output_dim=hidden_dim,init=init,
inner_init=inner_init, activation=activation,
inner_activation=inner_activation,weights=weights[encoder_index],
truncate_gradient = truncate_gradient, input_length=input_length,
hidden_state=hidden_state[encoder_index], batch_size=batch_size, remember_state=context_sensitive)
left_deep = [LSTMEncoder(input_dim=input_dim, output_dim=input_dim,init=init,
inner_init=inner_init, activation=activation,
inner_activation=inner_activation,weights=weights[i],
truncate_gradient = truncate_gradient, input_length=input_length,
hidden_state=hidden_state[i], batch_size=batch_size, return_sequences=True, remember_state=context_sensitive)
for i in range(depth[0]-1)]
right_deep = [LSTMEncoder(input_dim=output_dim, output_dim=output_dim,init=init,
inner_init=inner_init, activation=activation,
inner_activation=inner_activation,weights=weights[decoder_index + 1 + i],
truncate_gradient = truncate_gradient, input_length=input_length,
hidden_state=hidden_state[decoder_index + 1 + i], batch_size=batch_size, return_sequences=True, remember_state=context_sensitive)
for i in range(depth[1]-1)]
dense = Dense(input_dim=hidden_dim, output_dim=output_dim)
encoder.broadcast_state(decoder)
if weights[depth[0]] is not None:
dense.set_weights(weights[depth[0]])
super(Seq2seq, self).__init__()
for l in left_deep:
self.add(l)
self.add(encoder)
self.add(dense)
self.add(decoder)
for l in right_deep:
self.add(l)
self.encoder = encoder
self.dense = dense
self.decoder = decoder
self.left_deep = left_deep
self.right_deep = right_deep
print(
"------------------------------------------------------\nSetting up color ANN...\n"
)
model_color = Sequential()
model_color.add(
Conv2D(1,
filter_size,
padding='same',
activation='relu',
input_shape=(image_size[0], image_size[1], 3)))
model_color.add(BatchNormalization())
model_color.add(MaxPooling2D(pool_size=2))
model_color.add(Flatten())
model_color.add(Dense(number_dense_color, activation='relu'))
model_color.add(BatchNormalization())
model_color.add(Dense(color_output_size, activation='softmax'))
print(
"------------------------------------------------------\nTraining color ANN...\n"
)
plot_model(model_color,
to_file='models/color/color_ANN_model.png',
show_shapes=True,
show_layer_names=False)
coloroptimizer = SGD(lr=lr_inicial_color,
decay=lr_inicial_color / Epochs_color)
model_color.compile(loss="categorical_crossentropy",
optimizer=coloroptimizer,
class DeepLSTM(Recurrent):
'''Seq2Seq Deep Long-Short Term Memory unit.
Inspired byt Sutskever et. al 2014
This layer outputs ALL the states and cells like [h_0, c_0, ..., h_deeper, c_deeper].
If you need only the very last states, use a Lambada layer to narrow
output[:, -2*output_dim:-output_dim]
Args: similar to regular LSTM
depth: number of LSTMs to stack
readout: int, if we should a final Dense layer on top or not. readout is
this Dense's output_dim
'''
def __init__(self, output_dim, depth=1, readout=False, dropout=.5,
init='glorot_uniform', inner_init='orthogonal',
forget_bias_init='one', activation='tanh',
inner_activation='hard_sigmoid', **kwargs):
self.output_dim = output_dim
self.depth = depth
self.readout = readout
self.dropout = dropout
self.init = initializations.get(init)
self.inner_init = initializations.get(inner_init)
self.forget_bias_init = initializations.get(forget_bias_init)
self.activation = activations.get(activation)
self.inner_activation = activations.get(inner_activation)
self._kwargs = kwargs
super(DeepLSTM, self).__init__(**kwargs)
def build(self):
self.lstms = []
for i in range(self.depth):
if i == 0:
self.lstms.append(LSTM(self.output_dim, self.init, self.inner_init,
self.forget_bias_init, self.activation,
self.inner_activation, **self._kwargs))
else:
self._kwargs['input_dim'] = self.output_dim
self.lstms.append(LSTM(self.output_dim, self.init, self.inner_init,
self.forget_bias_init, self.activation,
self.inner_activation, **self._kwargs))
[lstm.build() for lstm in self.lstms]
# Get a flat list of trainable_weights
self.trainable_weights = [weights for lstm in self.lstms for weights in
lstm.trainable_weights]
if self.readout:
self.readout_layer = Dense(self.readout, input_dim=self.output_dim,
activation='softmax')
self.readout_layer.build()
self.trainable_weights.extend(self.readout_layer.trainable_weights)
if self.initial_weights is not None:
self.set_weights(self.initial_weights)
del self.initial_weights
if self.stateful is not None:
self.states = []
for lstm in self.lstms:
self.states.extend(lstm.states)
else:
self.states = [None, None] * self.depth
def reset_states(self):
[lstm.reset_states() for lstm in self.lstms]
def get_initial_states(self, X):
states = super(DeepLSTM, self).get_initial_states(X)
if self.readout:
initial_state = K.zeros_like(X) # (samples, timesteps, input_dim)
initial_state = K.sum(initial_state, axis=1) # (samples, input_dim)
reducer = K.zeros((self.input_dim, self.readout))
initial_state = K.dot(initial_state, reducer) # (samples, output_dim)
states += [initial_state]
return states
def step(self, x, states):
if self.readout:
assert len(states) == 2*self.depth+1
states = states[:-1]
else:
assert len(states) == 2*self.depth
h = []
# P = Print('[debug] X value: ', attrs=("shape",))
for i, (h_tm1, c_tm1) in enumerate(zip(states[:-1:2], states[1::2])):
# x = P(x)
x, new_states = self.lstms[i].step(x, [h_tm1, c_tm1])
h.extend(new_states)
# x = K.dropout(x, self.dropout) # no dropout on the first layer inputs
if self.readout:
h += [self.readout_layer(h[-2])]
return K.concatenate(h, axis=-1), h
def dream(self, length=140):
def _dream_step(x, states):
# input + states
assert len(states) == 2*self.depth + 1
x = states[-1]
x = K.switch(K.equal(x, K.max(x, axis=-1,
keepdims=True)), 1., 0.)
states = states[:-1]
h = []
for i, (h_tm1, c_tm1) in enumerate(zip(states[:-1:2], states[1::2])):
x, new_states = self.lstms[i].step(x, [h_tm1, c_tm1])
h.extend(new_states)
if self.readout:
h += [self.readout_layer(h[-2])]
final = h[-1]
else:
h += [h[-2]]
final = h[-2]
return final, h
# input shape: (nb_samples, time (padded with zeros), input_dim)
# Only the very first time point of the input is used, the others only
# server to count the lenght of the output sequence
X = self.get_input(train=False)
mask = self.get_input_mask(train=False)
assert K.ndim(X) == 3
if K._BACKEND == 'tensorflow':
if not self.input_shape[1]:
raise Exception('When using TensorFlow, you should define ' +
'explicitly the number of timesteps of ' +
'your sequences.\n' +
'If your first layer is an Embedding, ' +
'make sure to pass it an "input_length" ' +
'argument. Otherwise, make sure ' +
'the first layer has ' +
'an "input_shape" or "batch_input_shape" ' +
'argument, including the time axis.')
# if self.stateful:
# initial_states = self.states
# else:
# initial_states = self.get_initial_states(X)
s = self.get_output(train=False)[:, -1]
idx = [0, ] + list(np.cumsum([self.output_dim]*2*self.depth +
[self.readout, ]))
initial_states = [s[:, idx[i]:idx[i+1]] for i in range(len(idx)-1)]
# if self.readout:
# initial_states.pop(-1)
# initial_states.append(X[:, 0])
last_output, outputs, states = K.rnn(_dream_step, K.zeros((1, length, 1)),
initial_states,
go_backwards=self.go_backwards,
mask=mask)
if self.stateful:
self.updates = []
for i in range(len(states)):
self.updates.append((self.states[i], states[i]))
return outputs
def get_config(self):
config = {"output_dim": self.output_dim,
"depth": self.depth,
"readout": self.readout,
"dropout": self.dropout,
"init": self.init.__name__,
"inner_init": self.inner_init.__name__,
"forget_bias_init": self.forget_bias_init.__name__,
"activation": self.activation.__name__,
"inner_activation": self.inner_activation.__name__}
base_config = super(LSTM, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
@property
def output_shape(self):
input_shape = self.input_shape
if self.readout:
return input_shape[:2] + [self.output_dim*2*self.depth +
self.readout]
else:
return input_shape[:2] + tuple([self.output_dim*2*self.depth])
def dense_bn(n):
return [Dense(n), batch_norm(mode=1), Activation('relu')]
def lstm_model(data,
hidden_layer_neurons,
epochs,
feature_dimensions=1,
verbose=False):
"""Build an LSTM model.
Args:
data: Data frame of X, Y values
hidden_layer_neurons: Number of neurons per layers
epochs: Number of iterations for learning
feature_dimensions: Dimension of features (Number of rows per feature)
Returns:
model: Graph of LSTM model
"""
# Initialize key variables
start = time.time()
"""
In a stateful LSTM network, you should only pass inputs with a number of
samples that can be divided by the batch size. Hence we use "1" as it is a
factor in any possible number of samples.
"""
batch_size = 1
# Process the data for fitting
x_values, y_values = data[:, 0:-1], data[:, -1]
x_shaped = x_values.reshape(x_values.shape[0], 1, x_values.shape[1])
# Let's do some learning!
model = Sequential()
"""
The Long Short-Term Memory network (LSTM) is a type of Recurrent Neural
Network (RNN).
A benefit of this type of network is that it can learn and remember over
long sequences and does not rely on a pre-specified window lagged
observation as input.
In Keras, this is referred to as being "stateful", and involves setting the
"stateful" argument to "True" when defining an LSTM layer.
By default, an LSTM layer in Keras maintains state between data within
one batch. A batch of data is a fixed-sized number of rows from the
training dataset that defines how many patterns (sequences) to process
before updating the weights of the network.
A state is:
Where am I now inside a sequence? Which time step is it? How is this
particular sequence behaving since its beginning up to now?
A weight is: What do I know about the general behavior of all sequences
I've seen so far?
State in the LSTM layer between batches is cleared by default. This is
undesirable therefore we must make the LSTM stateful. This gives us
fine-grained control over when state of the LSTM layer is cleared, by
calling the reset_states() function during the model.fit() method.
LSTM networks can be stacked in Keras in the same way that other layer
types can be stacked. One addition to the configuration that is required
is that an LSTM layer prior to each subsequent LSTM layer must return the
sequence. This can be done by setting the return_sequences parameter on
the layer to True.
batch_size denotes the subset size of your training sample (e.g. 100 out
of 1000) which is going to be used in order to train the network during its
learning process. Each batch trains network in a successive order, taking
into account the updated weights coming from the appliance of the previous
batch.
return_sequence indicates if a recurrent layer of the network should return
its entire output sequence (i.e. a sequence of vectors of specific
dimension) to the next layer of the network, or just its last only output
which is a single vector of the same dimension. This value can be useful
for networks conforming with an RNN architecture.
batch_input_shape defines that the sequential classification of the
neural network can accept input data of the defined only batch size,
restricting in that way the creation of any variable dimension vector.
It is widely used in stacked LSTM networks. It is a tuple of (batch_size,
timesteps, data_dimension)
"""
timesteps = x_shaped.shape[1]
data_dimension = x_shaped.shape[2]
# Add layers to the model
model.add(
LSTM(units=hidden_layer_neurons,
batch_input_shape=(batch_size, timesteps, data_dimension),
return_sequences=True,
stateful=True))
model.add(Dropout(0.2))
model.add(
LSTM(units=hidden_layer_neurons,
batch_input_shape=(batch_size, timesteps, data_dimension),
return_sequences=False,
stateful=True))
model.add(Dropout(0.2))
model.add(Dense(units=feature_dimensions))
# model.add(Activation('linear'))
"""
Once the network is specified, it must be compiled into an efficient
symbolic representation using a backend mathematical library,
such as TensorFlow.
In compiling the network, we must specify a loss function and optimization
algorithm. We will use "mean_squared_error" or "mse" as the loss function
as it closely matches RMSE that we will are interested in, and the
efficient ADAM optimization algorithm.
"""
model.compile(loss='mse', optimizer='adam', metrics=['accuracy'])
"""
Once the model is compiled, the network can be fit to the training data.
Because the network is stateful, we must control when the internal state
is reset. Therefore, we must manually manage the training process one epoch
at a time across the desired number of epochs.
By default, the samples within an epoch are shuffled prior to being exposed
to the network. Again, this is undesirable for the LSTM because we want the
network to build up state as it learns across the sequence of observations.
We can disable the shuffling of samples by setting "shuffle" to "False".
"""
for _ in range(epochs):
model.fit(x_shaped,
y_values,
batch_size=batch_size,
shuffle=False,
epochs=1,
verbose=verbose,
validation_split=0.05)
"""
When the fit process reaches the total length of the samples,
model.reset_states() is called to reset the internal state at the end
of the training epoch, ready for the next training iteration.
This iteration will start training from the beginning of the dataset
therefore state will need to be reset as the previous state would only
be relevant to the prior epoch iteration.
"""
model.reset_states()
print('\n> Training Time: {:20.2f}'.format(time.time() - start))
return model
def build(width, height, depth, classes):
model = Sequential()
inputShape = (height, width, depth)
chanDim = -1
if K.image_data_format() == "channels_first":
inputShape = (depth, height, width)
chanDim = 1
model.add(
Conv2D(32, (3, 3),
padding="same",
kernel_initializer="he_normal",
input_shape=inputShape))
model.add(ELU())
model.add(BatchNormalization(axis=chanDim))
model.add(
Conv2D(32, (3, 3), kernel_initializer="he_normal", padding="same"))
model.add(ELU())
model.add(BatchNormalization(axis=chanDim))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(
Conv2D(64, (3, 3), kernel_initializer="he_normal", padding="same"))
model.add(ELU())
model.add(BatchNormalization(axis=chanDim))
model.add(
Conv2D(64, (3, 3), kernel_initializer="he_normal", padding="same"))
model.add(ELU())
model.add(BatchNormalization(axis=chanDim))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(
Conv2D(128, (3, 3), kernel_initializer="he_normal",
padding="same"))
model.add(ELU())
model.add(BatchNormalization(axis=chanDim))
model.add(
Conv2D(128, (3, 3), kernel_initializer="he_normal",
padding="same"))
model.add(ELU())
model.add(BatchNormalization(axis=chanDim))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(64, kernel_initializer="he_normal"))
model.add(ELU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(64, kernel_initializer="he_normal"))
model.add(ELU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(classes, kernel_initializer="he_normal"))
model.add(Activation("softmax"))
return model
def tag3d_network_dense(input,
nb_units=64,
nb_dense_units=[512, 512],
depth=2,
nb_output_channels=1,
trainable=True):
n = nb_units
def conv(n, repeats=None):
def normal(shape, name=None):
return keras.initializations.normal(shape, scale=0.01, name=name)
if repeats is None:
repeats = depth
return [[
Convolution2D(n, 3, 3, border_mode='same', init='he_normal'),
Activation('relu')
] for _ in range(repeats)]
base = sequential(
[
[
Dense(nb_dense, activation='relu')
for nb_dense in nb_dense_units
],
Dense(8 * n * 4 * 4),
Activation('relu'),
Reshape((
8 * n,
4,
4,
)),
conv(8 * n),
UpSampling2D(), # 8x8
conv(4 * n),
UpSampling2D(), # 16x16
conv(2 * n),
],
ns='tag3d_gen.base',
trainable=trainable)(input)
tag3d = sequential(
[
conv(2 * n),
UpSampling2D(), # 32x32
conv(n),
UpSampling2D(), # 64x64
conv(n, 1),
Convolution2D(1, 3, 3, border_mode='same', init='he_normal'),
],
ns='tag3d',
trainable=trainable)(base)
depth_map = sequential([
conv(n // 2, depth - 1),
Convolution2D(1, 3, 3, border_mode='same', init='he_normal'),
],
ns='depth_map',
trainable=trainable)(base)
return name_tensor(tag3d, 'tag3d'), name_tensor(depth_map, 'depth_map')
var_exp = [(i / np.sum(eigenvalues)) * 100 for i in eigenvalues]
cum_var_exp = np.cumsum(var_exp)
# print 'importance of each eigenvalue'
# print var_exp
# print 'importance of sum of eigenvalues'
# print cum_var_exp
accuracies_folds_withoutPCA = []
#withoutPCA
for train, test in kf.split(X_train.as_matrix()):
kX_train2, kX_test2, kY_train2, kY_test2 = X_train.as_matrix(
)[train], X_train.as_matrix()[test], Y_train[train], Y_train[test]
model = Sequential()
model.add(Dense(8, input_shape=(kX_train2.shape[1:])))
#model.add(Dropout(0.2))
#hidden layers
# model.add(Dense(64, activation='relu')) #,W_constraint=maxnorm(1)
# model.add(Dropout(0.2))
model.add(Dense(7, activation='relu')) #,W_constraint=maxnorm(1)
# model.add(Dropout(0.5))
#output layer
model.add(Dense(2, activation='softmax'))
# model.summary()
# Compile model
sgd = SGD(lr=learning_rate, momentum=0.7, nesterov=True)
# adam=Adam(lr=learning_rate, beta_1=0.7, beta_2=0.999, epsilon=1e-08, decay=0.0000001)
