def __init__(self):
super(HasTuple, self).__init__()
self.layer_list = (core.Dense(3), core.Dense(4),
core.Dense(5, kernel_regularizer=tf.reduce_sum))
self.layers_with_updates = (normalization.BatchNormalization(), )
def _model_compile(self,
strategy,
steps_per_execution=1,
run_eagerly=False,
with_normalization_layer=False):
class ResultAssertingCallback(callbacks_lib.Callback):
def __init__(self):
self._prev_epoch = -1
self._loss_to_compare_against = 2 # Empirical initial value
def on_epoch_end(self, epoch, logs=None):
logging.info("testModelFit: epoch=%r, logs=%r", epoch, logs)
if epoch <= self._prev_epoch:
raise RuntimeError(
"Epoch is supposed to be larger than previous.")
self._prev_epoch = epoch
is_loss_float = (logs.get("loss", None) is not None
and isinstance(logs["loss"],
(float, np.floating)))
if not is_loss_float:
raise RuntimeError(
"loss is supposed to be in the logs and float.")
if epoch == 0 or epoch == 9:
# Making sure the loss of first epoch is below 1, and that of last
# epoch is smaller than the first epoch.
if logs["loss"] > self._loss_to_compare_against:
raise RuntimeError(
"loss at epoch {} is larger than previous.".format(
epoch))
self._loss_to_compare_against = logs["loss"]
def on_train_end(self, logs=None):
if self._prev_epoch != 9:
raise RuntimeError("Unexpected last epoch: {}".format(
self._prev_epoch))
# TODO(b/182193218): Use ParameterServerStrategy as a proper strategy
# combination.
if strategy == "ParameterServerStrategy":
gpu_devices = tf.config.list_physical_devices("GPU")
if len(gpu_devices) > 1:
self.skipTest("b/178452835: Multi-GPUs not supported in "
"ParameterServerStrategy.")
strategy = tf.distribute.experimental.ParameterServerStrategy(
multi_worker_testing_utils.make_parameter_server_cluster(3, 2),
variable_partitioner=tf.distribute.experimental.partitioners.
FixedShardsPartitioner(2))
with strategy.scope():
model = sequential.Sequential([core_layers.Dense(10)])
if with_normalization_layer:
norm = keras.layers.BatchNormalization(axis=-1,
input_shape=(4, 4, 3),
momentum=0.8)
model.add(norm)
model.add(core_layers.Dense(1, activation="sigmoid"))
self._metric = keras.metrics.Accuracy()
model.compile(gradient_descent.SGD(),
loss="binary_crossentropy",
metrics=[self._metric],
steps_per_execution=steps_per_execution,
run_eagerly=run_eagerly)
return model, [ResultAssertingCallback()]
def test_dense():
layer = core.Dense(10, input_shape=(10,))
_runner(layer)
loc_model = MODELS.Sequential()
model = MODELS.Sequential()
if conv1:
conv_model.add(
CONV.Convolution2D(conv1_filters,
conv1_filter_size,
conv1_filter_size,
subsample=(conv1_stride, conv1_stride),
border_mode='valid',
input_shape=(prev_frames, image_size, image_size)))
if pool1:
conv_model.add(CONV.MaxPooling2D(pool_size=(pool1_size, pool1_size)))
conv_model.add(CORE.Activation(conv1_act))
conv_model.add(CORE.Flatten())
conv_model.add(CORE.Dense(fc1_size))
conv_model.add(CORE.Activation(fc1_act))
loc_model.add(CORE.Dense(fc1_size, input_shape=(prev_frames * 4, )))
loc_model.add(CORE.Activation(fc1_act))
#model.add(CONV.Convolution2D(conv2_filters, conv2_filter_size, conv2_filter_size, border_mode='valid'))
#model.add(CONV.MaxPooling2D(pool_size=(pool2_size, pool2_size)))
#model.add(CORE.Activation(conv2_act))
model.add(CORE.Merge([conv_model, loc_model], mode='concat'))
model.add(CORE.Dense(4, init='zero'))
model.add(CORE.Activation(fc2_act))
print 'Building bouncing MNIST generator'
from data_handler import *
bmnist = BouncingMNIST(1,
def testLayerCollectionWithExternalMutation(self): l = [] l_wrapper = data_structures.ListWrapper(l) layer = core.Dense(1) l.append(layer) self.assertEqual([layer], l_wrapper.layers)
def test_dense(self):
layer = core.Dense(10, 10)
self._runner(layer)
# Set no of x bits
numInputs = 3
y = [0, 1, 1, 0, 1, 0, 0, 1]
yClasses = kutils.to_categorical(y)
# Params
epochs = 10000
numX = len(X)
numClasses = 2 # Force binary classification
model = kmodels.Sequential()
model.add(core.Dense(
4,
input_shape=(numInputs, ),
activation="sigmoid",
))
model.add(core.Dense(numClasses, activation="sigmoid"))
model.summary()
model.compile(optimizer="adam", loss="mse")
model.fit(
X,
yClasses,
batch_size=numX,
nb_epoch=epochs,
show_accuracy=True,
)
class LayerCorrectnessTest(keras_parameterized.TestCase):
def setUp(self):
super(LayerCorrectnessTest, self).setUp()
# Set two virtual CPUs to test MirroredStrategy with multiple devices
cpus = tf.config.list_physical_devices('CPU')
tf.config.set_logical_device_configuration(cpus[0], [
tf.config.LogicalDeviceConfiguration(),
tf.config.LogicalDeviceConfiguration(),
])
def _create_model_from_layer(self, layer, input_shapes):
inputs = [layers.Input(batch_input_shape=s) for s in input_shapes]
if len(inputs) == 1:
inputs = inputs[0]
y = layer(inputs)
model = models.Model(inputs, y)
model.compile('sgd', 'mse')
return model
@parameterized.named_parameters(
('LeakyReLU', advanced_activations.LeakyReLU, (2, 2)),
('PReLU', advanced_activations.PReLU, (2, 2)),
('ELU', advanced_activations.ELU, (2, 2)),
('ThresholdedReLU', advanced_activations.ThresholdedReLU, (2, 2)),
('Softmax', advanced_activations.Softmax, (2, 2)),
('ReLU', advanced_activations.ReLU, (2, 2)),
('Conv1D', lambda: convolutional.Conv1D(2, 2), (2, 2, 1)),
('Conv2D', lambda: convolutional.Conv2D(2, 2), (2, 2, 2, 1)),
('Conv3D', lambda: convolutional.Conv3D(2, 2), (2, 2, 2, 2, 1)),
('Conv2DTranspose', lambda: convolutional.Conv2DTranspose(2, 2),
(2, 2, 2, 2)),
('SeparableConv2D', lambda: convolutional.SeparableConv2D(2, 2),
(2, 2, 2, 1)),
('DepthwiseConv2D', lambda: convolutional.DepthwiseConv2D(2, 2),
(2, 2, 2, 1)),
('UpSampling2D', convolutional.UpSampling2D, (2, 2, 2, 1)),
('ZeroPadding2D', convolutional.ZeroPadding2D, (2, 2, 2, 1)),
('Cropping2D', convolutional.Cropping2D, (2, 3, 3, 1)),
('ConvLSTM2D',
lambda: convolutional_recurrent.ConvLSTM2D(4, kernel_size=(2, 2)),
(4, 4, 4, 4, 4)),
('Dense', lambda: core.Dense(2), (2, 2)),
('Dropout', lambda: core.Dropout(0.5), (2, 2)),
('SpatialDropout2D', lambda: core.SpatialDropout2D(0.5), (2, 2, 2, 2)),
('Activation', lambda: core.Activation('sigmoid'), (2, 2)),
('Reshape', lambda: core.Reshape((1, 4, 1)), (2, 2, 2)),
('Permute', lambda: core.Permute((2, 1)), (2, 2, 2)),
('Attention', dense_attention.Attention, [(2, 2, 3), (2, 3, 3),
(2, 3, 3)]),
('AdditiveAttention', dense_attention.AdditiveAttention, [(2, 2, 3),
(2, 3, 3),
(2, 3, 3)]),
('Embedding', lambda: embeddings.Embedding(4, 4),
(2, 4), 2e-3, 2e-3, np.random.randint(4, size=(2, 4))),
('LocallyConnected1D', lambda: local.LocallyConnected1D(2, 2), (2, 2, 1)),
('LocallyConnected2D', lambda: local.LocallyConnected2D(2, 2),
(2, 2, 2, 1)),
('Add', merge.Add, [(2, 2), (2, 2)]),
('Subtract', merge.Subtract, [(2, 2), (2, 2)]),
('Multiply', merge.Multiply, [(2, 2), (2, 2)]),
('Average', merge.Average, [(2, 2), (2, 2)]),
('Maximum', merge.Maximum, [(2, 2), (2, 2)]),
('Minimum', merge.Minimum, [(2, 2), (2, 2)]),
('Concatenate', merge.Concatenate, [(2, 2), (2, 2)]),
('Dot', lambda: merge.Dot(1), [(2, 2), (2, 2)]),
('GaussianNoise', lambda: noise.GaussianNoise(0.5), (2, 2)),
('GaussianDropout', lambda: noise.GaussianDropout(0.5), (2, 2)),
('AlphaDropout', lambda: noise.AlphaDropout(0.5), (2, 2)),
('BatchNormalization', batch_normalization.BatchNormalization,
(2, 2), 1e-2, 1e-2),
('LayerNormalization', layer_normalization.LayerNormalization, (2, 2)),
('LayerNormalizationUnfused',
lambda: layer_normalization.LayerNormalization(axis=1), (2, 2, 2)),
('MaxPooling2D', pooling.MaxPooling2D, (2, 2, 2, 1)),
('AveragePooling2D', pooling.AveragePooling2D, (2, 2, 2, 1)),
('GlobalMaxPooling2D', pooling.GlobalMaxPooling2D, (2, 2, 2, 1)),
('GlobalAveragePooling2D', pooling.GlobalAveragePooling2D, (2, 2, 2, 1)),
('SimpleRNN', lambda: recurrent.SimpleRNN(units=4),
(4, 4, 4), 1e-2, 1e-2),
('GRU', lambda: recurrent.GRU(units=4), (4, 4, 4)),
('LSTM', lambda: recurrent.LSTM(units=4), (4, 4, 4)),
('GRUV2', lambda: recurrent_v2.GRU(units=4), (4, 4, 4)),
('LSTMV2', lambda: recurrent_v2.LSTM(units=4), (4, 4, 4)),
('TimeDistributed', lambda: wrappers.TimeDistributed(core.Dense(2)),
(2, 2, 2)),
('Bidirectional',
lambda: wrappers.Bidirectional(recurrent.SimpleRNN(units=4)), (2, 2, 2)),
('AttentionLayerCausal', lambda: dense_attention.Attention(causal=True), [
(2, 2, 3), (2, 3, 3), (2, 3, 3)
]),
('AdditiveAttentionLayerCausal',
lambda: dense_attention.AdditiveAttention(causal=True), [(2, 3, 4),
(2, 3, 4),
(2, 3, 4)]),
('NormalizationAdapt', _create_normalization_layer_with_adapt, (4, 4)),
('NormalizationNoAdapt', _create_normalization_layer_without_adapt,
(4, 4)),
('Resizing', lambda: image_preprocessing.Resizing(3, 3), (2, 5, 5, 1)),
('Rescaling', lambda: image_preprocessing.Rescaling(2., 1.), (6, 6)),
('CenterCrop', lambda: image_preprocessing.CenterCrop(3, 3),
(2, 5, 5, 1))
)
def test_layer(self, f32_layer_fn, input_shape, rtol=2e-3, atol=2e-3,
input_data=None):
"""Tests a layer by comparing the float32 and mixed precision weights.
A float32 layer, a mixed precision layer, and a distributed mixed precision
layer are run. The three layers are identical other than their dtypes and
distribution strategies. The outputs after predict() and weights after fit()
are asserted to be close.
Args:
f32_layer_fn: A function returning a float32 layer. The other two layers
will automatically be created from this
input_shape: The shape of the input to the layer, including the batch
dimension. Or a list of shapes if the layer takes multiple inputs.
rtol: The relative tolerance to be asserted.
atol: The absolute tolerance to be asserted.
input_data: A Numpy array with the data of the input. If None, input data
will be randomly generated
"""
if f32_layer_fn == convolutional.ZeroPadding2D and \
tf.test.is_built_with_rocm():
return
if isinstance(input_shape[0], int):
input_shapes = [input_shape]
else:
input_shapes = input_shape
strategy = create_mirrored_strategy()
f32_layer = f32_layer_fn()
# Create the layers
assert f32_layer.dtype == f32_layer._compute_dtype == 'float32'
config = f32_layer.get_config()
config['dtype'] = policy.Policy('mixed_float16')
mp_layer = f32_layer.__class__.from_config(config)
distributed_mp_layer = f32_layer.__class__.from_config(config)
# Compute per_replica_input_shapes for the distributed model
global_batch_size = input_shapes[0][0]
assert global_batch_size % strategy.num_replicas_in_sync == 0, (
'The number of replicas, %d, does not divide the global batch size of '
'%d' % (strategy.num_replicas_in_sync, global_batch_size))
per_replica_batch_size = (
global_batch_size // strategy.num_replicas_in_sync)
per_replica_input_shapes = [(per_replica_batch_size,) + s[1:]
for s in input_shapes]
# Create the models
f32_model = self._create_model_from_layer(f32_layer, input_shapes)
mp_model = self._create_model_from_layer(mp_layer, input_shapes)
with strategy.scope():
distributed_mp_model = self._create_model_from_layer(
distributed_mp_layer, per_replica_input_shapes)
# Set all model weights to the same values
f32_weights = f32_model.get_weights()
mp_model.set_weights(f32_weights)
distributed_mp_model.set_weights(f32_weights)
# Generate input data
if input_data is None:
# Cast inputs to float16 to avoid measuring error from having f16 layers
# cast to float16.
input_data = [np.random.normal(size=s).astype('float16')
for s in input_shapes]
if len(input_data) == 1:
input_data = input_data[0]
# Assert all models have close outputs.
f32_output = f32_model.predict(input_data)
mp_output = mp_model.predict(input_data)
self.assertAllClose(
mp_output, f32_output, rtol=rtol, atol=atol)
self.assertAllClose(
distributed_mp_model.predict(input_data), f32_output, rtol=rtol,
atol=atol)
# Run fit() on models
output = np.random.normal(size=f32_model.outputs[0].shape).astype('float16')
for model in f32_model, mp_model, distributed_mp_model:
model.fit(input_data, output, batch_size=global_batch_size)
# Assert all models have close weights
f32_weights = f32_model.get_weights()
self.assertAllClose(
mp_model.get_weights(), f32_weights, rtol=rtol, atol=atol)
self.assertAllClose(
distributed_mp_model.get_weights(), f32_weights, rtol=rtol, atol=atol)
def test_regularizers():
model = Sequential()
model.add(wrappers.TimeDistributed(core.Dense(2, kernel_regularizer='l1'), input_shape=(3, 4)))
model.add(core.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
assert len(model.losses) == 1
[vocab, vocab_size] = prepareQuestionVocab(questionsFile, threshQ, gloveFile)
[Xtrain, Xval] = prepareQuestionData(vocab, questionsFile, questionsFileVal)
ans_vocab = prepareAnswerVocab(answersFile, threshA)
[y_train, y_val] = prepareAnswerData(ans_vocab, answersFile, answersFileVal)
[x_train, x_val] = prepareImageData(248349, 121512, image_index_File,
image_index_FileVal)
model = loadCNNModel()
textmodel = loadTextModel(vocab_size, MAX_SEQUENCE_LENGTH)
merged = mergeModel(model, textmodel)
finalmodel = Sequential()
finalmodel.add(merged)
#finalmodel.add(textmodel)
#finalmodel.add(model)
finalmodel.add(core.Dense(4096, activation="relu"))
finalmodel.add(Dropout(0.5))
finalmodel.add(core.Dense(len(ans_vocab), activation="softmax"))
finalmodel.summary()
#sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
finalmodel.compile(loss="categorical_crossentropy",
optimizer="adadelta",
metrics=["accuracy"])
checkpointer = ModelCheckpoint(
filepath="../models/weights_prefeat_lstm_glove.hdf5",
verbose=1,
save_best_only=True)
tmp = finalmodel.fit([x_train, Xtrain],
def test_TimeDistributed():
# first, test with Dense layer
model = Sequential()
model.add(wrappers.TimeDistributed(core.Dense(2), input_shape=(3, 4)))
model.add(core.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 2)), epochs=1, batch_size=10)
# test config
model.get_config()
# test when specifying a batch_input_shape
test_input = np.random.random((1, 3, 4))
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(wrappers.TimeDistributed(core.Dense(2), batch_input_shape=(1, 3, 4)))
reference.add(core.Activation('relu'))
reference.compile(optimizer='rmsprop', loss='mse')
reference.layers[0].set_weights(weights)
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Embedding
model = Sequential()
model.add(wrappers.TimeDistributed(embeddings.Embedding(5, 6), batch_input_shape=(10, 3, 4), dtype='int32'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.randint(5, size=(10, 3, 4), dtype='int32'), np.random.random((10, 3, 4, 6)), epochs=1, batch_size=10)
# compare to not using batch_input_shape
test_input = np.random.randint(5, size=(10, 3, 4), dtype='int32')
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(wrappers.TimeDistributed(embeddings.Embedding(5, 6), input_shape=(3, 4), dtype='int32'))
reference.compile(optimizer='rmsprop', loss='mse')
reference.layers[0].set_weights(weights)
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Conv2D
model = Sequential()
model.add(wrappers.TimeDistributed(convolutional.Conv2D(5, (2, 2), padding='same'), input_shape=(2, 4, 4, 3)))
model.add(core.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.random.random((1, 2, 4, 4, 3)), np.random.random((1, 2, 4, 4, 5)))
model = model_from_json(model.to_json())
model.summary()
# test stacked layers
model = Sequential()
model.add(wrappers.TimeDistributed(core.Dense(2), input_shape=(3, 4)))
model.add(wrappers.TimeDistributed(core.Dense(3)))
model.add(core.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 3)), epochs=1, batch_size=10)
# test wrapping Sequential model
model = Sequential()
model.add(core.Dense(3, input_dim=2))
outer_model = Sequential()
outer_model.add(wrappers.TimeDistributed(model, input_shape=(3, 2)))
outer_model.compile(optimizer='rmsprop', loss='mse')
outer_model.fit(np.random.random((10, 3, 2)), np.random.random((10, 3, 3)), epochs=1, batch_size=10)
# test with functional API
x = Input(shape=(3, 2))
y = wrappers.TimeDistributed(model)(x)
outer_model = Model(x, y)
outer_model.compile(optimizer='rmsprop', loss='mse')
outer_model.fit(np.random.random((10, 3, 2)), np.random.random((10, 3, 3)), epochs=1, batch_size=10)
trainData = dataclean.convertPandasDataFrameToNumpyArray(trainFrame)
testFrame = dataclean.cleanDataset(dataclean.loadTestData(), True)
testData = dataclean.convertPandasDataFrameToNumpyArray(testFrame)
trainX = trainData[:, 1:]
trainY = trainData[:, 0]
testX = testData[:, 1:]
nFeatures = trainX.shape[1]
model = models.Sequential()
model.add(embed.Embedding(1, 256, input_length=nFeatures))
model.add(recurrent.LSTM(output_dim=128, activation="sigmoid"))
model.add(core.Dropout(0.2))
model.add(core.Dense(1))
model.summary()
model.compile(optimizer="sgd", loss="mse")
model.fit(
trainX,
trainY,
nb_epoch=100,
verbose=1,
)
finalPredicted = model.predict(testX)
for i, x in enumerate(finalPredicted):
finalPredicted[i] = finalPredicted[
i] if finalPredicted[i] >= 0 else -finalPredicted[i]
conv.Convolution2D(nb_filters_2,
nb_conv,
nb_conv,
activation="relu",
border_mode='same'))
cnn.add(conv.MaxPooling2D(strides=(2, 2)))
#cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same'))
#cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same'))
#cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same'))
#cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu", border_mode='same'))
#cnn.add(conv.MaxPooling2D(strides=(2,2)))
cnn.add(core.Flatten())
cnn.add(core.Dropout(0.2))
cnn.add(core.Dense(128, activation="relu")) # 4096
cnn.add(core.Dense(nb_classes, activation="softmax"))
cnn.summary()
cnn.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
cnn.fit(trainX, trainY, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1)
testX = test.reshape(test.shape[0], 28, 28, 1)
testX = testX.astype(float)
testX /= 255.0
yPred = cnn.predict_classes(testX)
x = x_in / 255
x = x.reshape(x.shape[0], 1, x.shape[1], x.shape[2])
return x
wrangledX = x_wrangle(rawX)
trainX = wrangledX[:n_train]
_, _, img_rows, img_cols = trainX.shape
# noise input
def gen_noise(batch_size, d):
return np.random.uniform(0, 1, size=[batch_size, d])
# generator
n_channels = 200
l_width = img_rows / 2
g_input = kl.Input(shape=[noise_size])
H = klc.Dense(n_channels*l_width*l_width, init='glorot_normal')(g_input)
H = kln.BatchNormalization(mode=2)(H)
H = klc.Activation('relu')(H)
H = klc.Reshape([n_channels, l_width, l_width])(H)
H = klconv.UpSampling2D(size=(2,2))(H)
H = klconv.Convolution2D(n_channels/2, 3, 3, border_mode='same',
init='glorot_uniform')(H)
H = kln.BatchNormalization(mode=2)(H)
H = klc.Activation('relu')(H)
H = klconv.Convolution2D(n_channels/4, 3, 3, border_mode='same',
init='glorot_uniform')(H)
H = kln.BatchNormalization(mode=2)(H)
H = klc.Activation('relu')(H)
H = klconv.Convolution2D(1, 1, 1, border_mode='same', init='glorot_uniform')(H)
g_V = klc.Activation('sigmoid')(H)
generator = km.Model(g_input, g_V)
def model(trainX, trainY, valX, valY):
'''
Model providing function
'''
model_callbacks = []
img_rows = 64
img_cols = 80
smooth = 1.
batch_size = 16
#passing argument 'test' I only train the model for 1 epoch
#passing argument 'epochN' (with N as a positive int) I train the model for N epochs
nb_epoch = 300
try:
nb_epoch = find_argument("epoch")
except ValueError:
pass
try:
find_argument("test")
nb_epoch = 1
except ValueError:
pass
act = 'relu'
base_layer_depth = 32
lmbda = 0.1
l2reg = l2(lmbda)
dropout = 0.5
opt = Adam() #Adadelta()
##transforming optimizer and parameters to string
optstr = str(opt.__class__).split(".")[2][:-2]
lr = opt.get_config()
lr = lr['lr']
optstr = optstr + '_lr-{0:.6g}'.format(lr)
pixel_offset = 2
### pixel_offset is converted into percentage compared to the image's pixel size
pixel_offset_w = pixel_offset / img_cols
pixel_offset_h = pixel_offset / img_rows
print "inputsize: " + str(img_rows) + ' ' + str(img_cols)
print "opt: " + str(optstr)
print "dropout: " + str(dropout)
print "batch_size: " + str(batch_size)
print "lambda l2 : " + str(lmbda)
print "pixel_offset : " + str(pixel_offset)
################### callbacks ###################
modelDir = 'models/logs_D-{0:.3f}'.format(
dropout) + '_o-' + optstr + '_lmd-' + str(lmbda) + '_px-' + str(
pixel_offset)
mkdir(modelDir)
early = EarlyStopping(monitor='val_loss',
patience=150,
verbose=1,
mode='auto')
#Callback to save the best epoch and, eventually, overwrite it if outperformed (regarding the same model)
checkpoint_name = modelDir + '/best_model.h5' #.{epoch:02d}-{val_loss:.4f}.h5'
checkpoint = ModelCheckpoint(checkpoint_name,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto')
#Tensorboard for each result
#tb_callback = TensorBoard(log_dir="./"+modelDir, histogram_freq=0, write_graph=True)
#WeightsGIF and ActivationsGIF
weigthsSave = WeightsGIF(modelDir, 1)
fileSave = FileMonitor(modelDir)
#activationsSave = ActivationsGIF(modelDir, 1, trainX[0])
#model_callbacks.append(tb_callback)
model_callbacks.append(checkpoint)
model_callbacks.append(early)
model_callbacks.append(weigthsSave)
model_callbacks.append(fileSave)
#model_callbacks.append(activationsSave)
################### Model and Layers definition ###################
image_input = Input((img_rows, img_cols, 3), name="images")
conv1 = Convolution2D(base_layer_depth,
5,
5,
activation='relu',
border_mode='same',
W_regularizer=l2reg,
b_regularizer=l2reg)(image_input)
conv1 = core.Dropout(dropout)(conv1)
conv1 = Convolution2D(base_layer_depth,
5,
5,
activation='relu',
border_mode='same',
W_regularizer=l2reg,
b_regularizer=l2reg)(conv1)
conv1 = core.Dropout(dropout)(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Convolution2D(base_layer_depth * 2,
3,
3,
activation='relu',
border_mode='same',
W_regularizer=l2reg,
b_regularizer=l2reg)(pool1)
conv2 = core.Dropout(dropout)(conv2)
conv2 = Convolution2D(base_layer_depth * 2,
3,
3,
activation='relu',
border_mode='same',
W_regularizer=l2reg,
b_regularizer=l2reg)(conv2)
conv2 = core.Dropout(dropout)(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Convolution2D(base_layer_depth * 4,
3,
3,
activation='relu',
border_mode='same',
W_regularizer=l2reg,
b_regularizer=l2reg)(pool2)
conv3 = core.Dropout(dropout)(conv3)
conv3 = Convolution2D(base_layer_depth * 4,
3,
3,
activation='relu',
border_mode='same',
W_regularizer=l2reg,
b_regularizer=l2reg)(conv3)
conv3 = core.Dropout(dropout)(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Convolution2D(base_layer_depth * 8,
3,
3,
activation='relu',
border_mode='same',
W_regularizer=l2reg,
b_regularizer=l2reg)(pool3)
conv4 = core.Dropout(dropout)(conv4)
conv4 = Convolution2D(base_layer_depth * 8,
3,
3,
activation='relu',
border_mode='same',
W_regularizer=l2reg,
b_regularizer=l2reg)(conv4)
conv4 = core.Dropout(dropout)(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
conv5 = Convolution2D(base_layer_depth * 16,
3,
3,
activation='relu',
border_mode='same',
W_regularizer=l2reg,
b_regularizer=l2reg)(pool4)
conv5 = core.Dropout(dropout)(conv5)
conv5 = Convolution2D(base_layer_depth * 16,
3,
3,
activation='relu',
border_mode='same',
W_regularizer=l2reg,
b_regularizer=l2reg)(conv5)
conv5 = core.Dropout(dropout)(conv5)
flat = core.Flatten()(conv5)
dense = core.Dense(256, activation='relu')(flat)
dense = core.Dense(16, activation='relu')(dense)
#Auxiliary Inputs
aux_inputs_list = []
for label in input_labels:
if not label == "images":
aux_inputs_list.append(
Input((trainX[label].shape[1], ), name=label))
inputs_list = [image_input]
for element in aux_inputs_list:
inputs_list.append(element)
merge_list = [dense] + aux_inputs_list
merge_layer = merge(merge_list,
mode='concat',
concat_axis=1,
name="merging")
dense_final = core.Dense(128, activation='relu',
name="final_1")(merge_layer)
dense_final = core.Dropout(dropout)(dense_final)
dense_final = core.Dense(64, activation='relu',
name="final_2")(dense_final)
dense_final = core.Dropout(dropout)(dense_final)
prediction = core.Dense(trainY.shape[1],
activation='softmax',
name="output")(dense_final)
model = Model(input=inputs_list, output=prediction)
model.summary()
model.compile(optimizer=opt,
loss='categorical_crossentropy',
metrics=['accuracy'])
hist = model.fit(trainX,
trainY,
batch_size=batch_size,
nb_epoch=nb_epoch,
verbose=1,
callbacks=model_callbacks,
validation_data=(valX, valY))
################### metrics reporting ###################
val_loss, val_acc = model.evaluate(valX, valY, verbose=0)
name_file_save = 'final_model'
keras_model_save(model, modelDir, name_file_save)
return {'loss': val_loss, 'status': STATUS_OK}
def __init__(self):
super().__init__(name="")
self.fc = keras_core.Dense(1,
name="fc",
kernel_initializer="ones",
bias_initializer="ones")
conv.Convolution2D(nb_filters_4,
nb_conv,
nb_conv,
activation="relu",
border_mode='same'))
cnn.add(
conv.Convolution2D(nb_filters_4,
nb_conv,
nb_conv,
activation="relu",
border_mode='same'))
cnn.add(conv.MaxPooling2D(strides=(2, 2)))
cnn.add(core.Flatten())
cnn.add(core.Dropout(0.2)) # mitigate overfitting
cnn.add(core.Dense(128,
activation="relu")) # 4096 fully-connected forward neuro
cnn.add(core.Dense(nb_classes, activation="softmax"))
cnn.summary()
cnn.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
cnn.fit(trainX, trainY, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1)
testX = test.reshape(test.shape[0], 28, 28, 1)
testX = testX.astype(float)
#testX /= 255.0
testX = standardize(testX)
yPred = cnn.predict_classes(testX)
trainX = trainData[:, 1:].astype(np.float32) trainY = kutils.to_categorical(trainData[:, 0]) noFeatures = trainX.shape[1] scaler = preproc.StandardScaler() trainX = scaler.fit_transform(trainX) """ Final Model """ epochs = 8 nn = models.Sequential() nn.add(core.Dense(noFeatures, input_shape=(noFeatures, ))) nn.add(advact.PReLU()) nn.add(norm.BatchNormalization()) nn.add(core.Dropout(0.2)) nn.add(core.Dense(2 * noFeatures, )) nn.add(advact.PReLU()) nn.add(norm.BatchNormalization()) nn.add(core.Dropout(0.25)) nn.add(core.Dense(noFeatures, )) nn.add(advact.PReLU()) nn.add(norm.BatchNormalization()) nn.add(core.Dropout(0.2)) nn.add(core.Dense(noOfClasses, activation="softmax"))
random_state=1)
# numpy.unique returns a list of unique values in an array.
# When return_inverse is set to True it also returns the original array encoded
uniq_test, ids_test = np.unique(out_test, return_inverse=True)
uniq_train, ids_train = np.unique(out_train, return_inverse=True)
# the to_categorical function turns indices into bonary vectors.
#
onehot_train = u.np_utils.to_categorical(ids_train, len(uniq_test))
onehot_test = u.np_utils.to_categorical(ids_test, len(uniq_train))
# Create a model with a linear stack of layers with 16 nodes.
#
model = m.Sequential()
# Apply the sigmoid activation function to that layer.
model.add(c.Dense(16, input_dim=4))
model.add(c.Activation("sigmoid"))
# Add another layer, connected to the layer wth 16 nodes, containing three output nodees.
model.add(c.Dense(3)) # Dense = 密集
# Use the softmax activation function there
model.add(c.Activation("softmax"))
# Configure the model for training.
# Docs: https://keras.io/models/sequential/#compile
# Uses the adam optimizer and categorical cross entropy as the loss function
model.compile(optimizer="adam", loss="categorical_crossentropy")
# Fit the model using our training data.
# epochs , batch_size, verbose 分别什么作用？
model.fit(in_train, onehot_train, epochs=100, batch_size=1, verbose=1)
def __init__(self):
super(MyModel, self).__init__()
self._named_dense = core.Dense(1, use_bias=True)
self._second = core.Dense(1, use_bias=False)
# We can still track Trackables which aren't Layers.
self._non_layer = NonLayerTrackable()
cnn.add(conv.Convolution2D(nb_filters_2, nb_conv, nb_conv, activation="relu"))
cnn.add(conv.MaxPooling2D(strides=(2,2)))
cnn.add(conv.ZeroPadding2D((1, 1)))
cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu"))
cnn.add(conv.ZeroPadding2D((1, 1)))
cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu"))
cnn.add(conv.ZeroPadding2D((1, 1)))
cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu"))
cnn.add(conv.ZeroPadding2D((1, 1)))
cnn.add(conv.Convolution2D(nb_filters_3, nb_conv, nb_conv, activation="relu"))
cnn.add(conv.MaxPooling2D(strides=(2,2)))
cnn.add(core.Flatten())
cnn.add(core.Dropout(0.2))
cnn.add(core.Dense(1024, activation="relu"))
cnn.add(core.Dense(nb_classes, activation="softmax"))
cnn.summary()
cnn.compile(loss="categorical_crossentropy", optimizer="adadelta", metrics=["accuracy"])
#cnn.load_weights("VGG_Temp.h5")
#print("Model loaded.")
cnn.fit(trainX, trainY, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, validation_data=(validateX, validationY)) #validation_split=0.01,
#callbacks=[callbacks.ModelCheckpoint("VGG_Best.h5", save_best_only=True)]
print("Model fit.")
cnn.save_weights("VGG_Temp.h5", overwrite=True)
print("Weights Saved.")
def test_dense(self):
layer = core.Dense(10, input_shape=(10,))
self._runner(layer)
print('Making Questions Vocab')
[vocab, vocab_size] = prepareQuestionVocab(questionsFile, threshQ)
print('Making Answers Vocab')
ans_vocab = prepareAnswerVocab(answersFile, threshA)
print('Prepare Image Map')
[Idmap, mat] = prepareImageData()
print('Making Model')
model = loadCNNModel()
textmodel = loadTextModel(vocab_size, MAX_SEQUENCE_LENGTH)
merged = mergeModel(model, textmodel)
finalmodel = Sequential()
finalmodel.add(merged)
#finalmodel.add(textmodel)
#finalmodel.add(model)
finalmodel.add(core.Dense(4096, activation="relu", trainable=False))
finalmodel.add(Dropout(0.5))
finalmodel.add(
core.Dense(len(ans_vocab), activation="softmax", trainable=False))
print('Loading Wights')
finalmodel.load_weights(weightsFile)
#print ans_vocab
for i in range(30):
image_id = unicode(raw_input("Give the image Id:"))
#image_id=292181
question = unicode(raw_input("Ask a question about the image:"))
#question='What is in the picture'
print('Making Question Vector')
def testRunStepsWithOutputContext(self, distribution, optimizer_fn, is_tpu):
with distribution.scope():
def dataset_fn():
dataset = tf.data.Dataset.from_tensors([[1.]]).repeat()
# TODO(priyag): batch with drop_remainder=True causes shapes to be
# fully defined for TPU. Remove this when XLA supports dynamic shapes.
return dataset.batch(batch_size=1, drop_remainder=True)
optimizer = optimizer_fn()
layer = core.Dense(1, use_bias=True)
key1 = "foo"
value1 = "bar"
def model_fn(output_context, x):
"""A very simple model written by the user."""
def loss_fn():
y = tf.reshape(layer(x), []) - tf.constant(1.)
return y * y
if isinstance(optimizer, optimizer_v2.OptimizerV2):
train_op = optimizer.minimize(
loss_fn, lambda: layer.trainable_variables)
else:
train_op = optimizer.minimize(loss_fn)
loss = loss_fn()
output_context.set_last_step_output(
name="replica_loss_reduced",
output=loss,
reduce_op=tf.distribute.ReduceOp.MEAN)
output_context.set_non_tensor_output(key1, value1)
return (train_op, loss)
def step_fn(output_context, inputs):
(train_op, loss) = distribution.extended.call_for_each_replica(
model_fn, args=(output_context, inputs))
output_context.set_last_step_output(
name="cross_replica_loss_reduced",
output=loss,
reduce_op=tf.distribute.ReduceOp.MEAN)
output_context.set_last_step_output(
name="cross_replica_loss_not_reduced",
output=loss)
return distribution.group(train_op)
iterator = self._get_iterator(distribution, dataset_fn)
def run_step():
initial_loss = lambda: tf.constant(1e7)
# Initial values corresponding to reduced losses are just single
# tensors. But for non reduced losses, we need to have initial
# values that are of the same structure as non reduced losses. In
# MirroredStrategy, this will be a list of losses, in TPUStrategy
# it will be single tensor. Using `call_for_each_replica` followed
# by `experimental_local_results` gives us the desired initial
# value structure.
not_reduced = distribution.experimental_local_results(
distribution.extended.call_for_each_replica(initial_loss))
initial_loop_values = {
"replica_loss_reduced": initial_loss(),
"cross_replica_loss_reduced": initial_loss(),
"cross_replica_loss_not_reduced": not_reduced,
}
ctx = distribution.extended.experimental_run_steps_on_iterator(
step_fn, iterator, iterations=2,
initial_loop_values=initial_loop_values)
self.assertEqual({key1: (value1,)}, ctx.non_tensor_outputs)
self._verify_loss_output(
initial_loss(),
loss_output=ctx.last_step_outputs["replica_loss_reduced"],
reduced=True, distribution=distribution)
self._verify_loss_output(
initial_loss(),
loss_output=ctx.last_step_outputs["cross_replica_loss_reduced"],
reduced=True, distribution=distribution)
self._verify_loss_output(
initial_loss(),
loss_output=ctx.last_step_outputs["cross_replica_loss_not_reduced"],
reduced=False, distribution=distribution)
return (ctx.run_op, ctx.last_step_outputs["replica_loss_reduced"])
if not tf.executing_eagerly():
with self.cached_session() as sess:
run_step = sess.make_callable(run_step())
self.evaluate(tf.compat.v1.global_variables_initializer())
weights, biases = [], []
for _ in range(5):
run_step()
weights.append(self.evaluate(layer.kernel))
biases.append(self.evaluate(layer.bias))
error = abs(
numpy.add(numpy.squeeze(weights), numpy.squeeze(biases)) - 1)
error_is_not_increasing = all(y <= x for x, y in zip(error, error[1:]))
self.assertTrue(error_is_not_increasing)
trainX = trainData[:, 2:] trainX -= np.mean(trainX) trainX /= np.std(trainX) nFeatures = trainX.shape[1] trainY = trainData[:, 1] testX = testData[:, 1:] testX -= np.mean(testX) testX /= np.std(testX) epochs = 100 model = models.Sequential() model.add(core.Dense(2000, init="uniform", input_shape=(nFeatures,), activation="relu")) model.add(core.Dropout(0.2)) model.add(core.Dense(1000, activation="relu")) model.add(core.Dropout(0.2)) model.add(core.Dense(1000, activation="relu")) model.add(core.Dropout(0.2)) model.add(core.Dense(1000, activation="relu")) model.add(core.Dense(1, activation="sigmoid")) model.summary() model.compile(optimizer="adamax", loss="binary_crossentropy", class_mode="binary") model.fit(trainX, trainY, nb_epoch=epochs, validation_split=0.05, show_accuracy=True) yPred = model.predict_proba(testX)[:,0] #print(yPred)
def testLayerCollectionWithExternalMutation(self):
l = []
l_wrapper = tf.__internal__.tracking.wrap(l)
layer = core.Dense(1)
l.append(layer)
self.assertEqual([layer], l_wrapper.layers)
# Model architecture
model = models.Sequential()
model.add(
convolutional.Convolution2D(16,
3,
3,
input_shape=(32, 128, 3),
activation='relu'))
model.add(pooling.MaxPooling2D(pool_size=(2, 2)))
model.add(convolutional.Convolution2D(32, 3, 3, activation='relu'))
model.add(pooling.MaxPooling2D(pool_size=(2, 2)))
model.add(convolutional.Convolution2D(64, 3, 3, activation='relu'))
model.add(pooling.MaxPooling2D(pool_size=(2, 2)))
model.add(core.Flatten())
model.add(core.Dense(500, activation='relu'))
model.add(core.Dropout(.5))
model.add(core.Dense(100, activation='relu'))
model.add(core.Dropout(.25))
model.add(core.Dense(20, activation='relu'))
model.add(core.Dense(1))
model.compile(optimizer=optimizers.Adam(lr=1e-04),
loss='mean_squared_error')
history = model.fit_generator(
generate_samples(df_train, local_data_path),
samples_per_epoch=df_train.shape[0],
nb_epoch=30,
validation_data=generate_samples(df_valid,
local_data_path,
augment=False),
def test_serialize_dense(self):
dense = core.Dense(3)
dense(tf.constant([[4.]]))
round_trip = json.loads(
json.dumps(dense, default=json_utils.get_json_type))
self.assertEqual(3, round_trip["config"]["units"])
nb_classes = trainY.shape[1]
cnn = models.Sequential()
cnn.add(
conv.Convolution2D(nb_filters,
nb_conv,
nb_conv,
border_mode="valid",
input_shape=(1, 28, 28),
activation="relu"))
cnn.add(conv.MaxPooling2D())
cnn.add(conv.Convolution2D(nb_filters, nb_conv, nb_conv, activation="relu"))
cnn.add(conv.MaxPooling2D())
cnn.add(core.Dropout(0.25))
cnn.add(core.Flatten())
cnn.add(core.Dense(100, activation="relu")) # 500, tanh
#cnn.add(core.Dropout(0.15))
cnn.add(core.Dense(nb_classes, activation="softmax"))
#sgd = optm.sgd(lr=0.01, momentum=0.9, decay=1e-6, nesterov=True)
cnn.compile(
loss="mean_squared_error",
optimizer="adadelta",
)
cnn.fit(
trainX,
trainY,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,