def vgg_based_model(input_shape, n_categories, fulltraining=False):
"""
VGG16をベースにした転移学習モデルを生成する。
fulltraining: Trueにすると、ベースモデルも含めて訓練可能にする。訓練速度が非常に遅くなる。
"""
base_model = VGG16(weights='imagenet',
include_top=False,
input_tensor=Input(shape=input_shape))
#add new layers instead of FC networks
x = noise.GaussianNoise(0.1)(Input(shape=input_shape))
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024)(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Dropout(0.5)(x)
x = Dense(1024)(x)
x = noise.GaussianNoise(0.01)(x)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = Dropout(0.5)(x)
prediction = Dense(n_categories, activation='softmax')(x)
model = Model(inputs=base_model.input, outputs=prediction)
if not fulltraining:
# fix weights before VGG16 14layers
for layer in base_model.layers[:15]:
layer.trainable = False
return model
def new_model(self, fresh = False, compile = True):
self.model = Sequential()
drop_cl = core.Dropout
if self.l1 != 0 or self.l2 != 0:
regularizer = WeightRegularizer(l1=self.l1, l2=self.l2)
else:
regularizer = None
if self.enc_decs and not fresh:
for (i,enc) in enumerate(ae.layers[0].encoder for ae in self.enc_decs):
if self.drop_rate != 0:
self.model.add(drop_cl(self.drop_rate, input_shape=(self.layer_sizes[i],)))
if self.sigma_base != 0:
self.model.add(noise.GaussianNoise(self.sigma_base*(self.sigma_fact**-i)))
self.model.add(enc)
else:
for (i,(n_in, n_out)) in enumerate(zip(self.layer_sizes[:-1], self.layer_sizes[1:])):
if self.drop_rate != 0:
self.model.add(drop_cl(self.drop_rate, input_shape=(n_in,)))
if self.sigma_base != 0:
self.model.add(noise.GaussianNoise(self.sigma_base*(self.sigma_fact**-i)))
self.model.add(core.Dense(input_dim=n_in, output_dim=n_out, activation='sigmoid', W_regularizer=regularizer))
#TODO ?
self.model.add(core.Dense(input_dim=self.layer_sizes[-1]
,output_dim=len(phase_names)
,activation='softmax'
,W_regularizer=regularizer))
if compile:
self.model.compile(loss=self.cls_lss, optimizer=self.cls_opt)
def model(data,
hidden_layers,
hidden_neurons,
output_file,
validation_split=0.9):
train_n = int(validation_split * len(data))
batch_size = 50
train_data = data[:train_n, :]
val_data = data[train_n:, :]
#data: data_num * data_dim
input_sh = Input(shape=(data.shape[1], ))
encoded = Dense(data.shape[1],
activation='relu',
activity_regularizer=regularizers.activity_l1l2(
10e-5, 10e-5))(input_sh)
encoded = noise.GaussianNoise(0.2)(encoded)
for i in range(hidden_layers):
encoded = Dense(hidden_neurons[i],
activation='relu',
activity_regularizer=regularizers.activity_l1l2(
10e-5, 10e-5))(encoded)
encoded = noise.GaussianNoise(0.2)(encoded)
for j in range(hidden_layers - 1, -1, -1):
decoded = Dense(hidden_neurons[i],
activation='relu',
activity_regularizer=regularizers.activity_l1l2(
10e-5, 10e-5))(encoded)
decoded = Dense(data.shape[1], activation='sigmoid')(decoded)
autoencoder = Model(input=input_sh, output=decoded)
autoencoder.compile(optimizer='adadelta', loss='mse')
checkpointer = ModelCheckpoint(filepath='data/bestmodel' + output_file +
".hdf5",
verbose=1,
save_best_only=True)
earlystopper = EarlyStopping(monitor='val_loss', patience=15, verbose=1)
train_generator = DataGenerator(batch_size)
train_generator.fit(train_data, train_data)
val_generator = DataGenerator(batch_size)
val_generator.fit(val_data, val_data)
autoencoder.fit_generator(train_generator,
samples_per_epoch=len(train_data),
nb_epoch=100,
validation_data=val_generator,
nb_val_samples=len(val_data),
max_q_size=batch_size,
callbacks=[checkpointer, earlystopper])
enco = Model(input=input_sh, output=encoded)
enco.compile(optimizer='adadelta', loss='mse')
reprsn = enco.predict(data)
return reprsn
def new_model(self,
fresh=False,
compile=True,
use_dropout=None,
use_noise=None):
self.model = Sequential()
if not use_dropout is None:
self.mod_use_drop = self.drop_rate > 0 and use_dropout
if not use_noise is None:
self.mod_use_noise = self.sigma_base > 0 and use_noise
drop_cl = core.Dropout
if self.l1 != 0 or self.l2 != 0:
regularizer = WeightRegularizer(l1=self.l1, l2=self.l2)
else:
regularizer = None
if self.enc_decs and not fresh:
# The encoder may already have a noise and/or drop layer!
# But we don't know what kind so replace them
# the if len... stuff only works if dropout isn't used for encdecs without gaussian
for (i, enc) in enumerate(
layers[0 if len(layers) == 1 else 1]
for ae in self.enc_decs
for layers in (ae.layers[0].encoder.layers, )):
#if self.sigma_base != 0:
# Always add this even if 0 because the encdec might have had one
self.model.add(
noise.GaussianNoise(self.sigma_base *
(self.sigma_fact**-i),
input_shape=(self.layer_sizes[i], )))
self.model.add(enc)
if self.drop_rate > 0 and self.mod_use_drop:
self.model.add(drop_cl(self.drop_rate))
else:
for (i, (n_in, n_out)) in enumerate(
zip(self.layer_sizes[:-1], self.layer_sizes[1:])):
if self.sigma_base > 0 and self.mod_use_noise:
self.model.add(
noise.GaussianNoise(
self.sigma_base * (self.sigma_fact**-i),
input_shape=(self.layer_sizes[i], )))
self.model.add(
core.Dense(input_dim=n_in,
output_dim=n_out,
activation='sigmoid',
W_regularizer=regularizer))
if self.drop_rate > 0 and self.mod_use_drop:
self.model.add(
drop_cl(self.drop_rate, input_shape=(n_in, )))
#TODO ?
self.model.add(
core.Dense(input_dim=self.layer_sizes[-1],
output_dim=len(phase_names),
activation='softmax',
W_regularizer=regularizer))
if compile:
self.model.compile(loss=self.cls_lss, optimizer=self.cls_opt)
def encoder(node_num, hidden_layers, hidden_neurons):
x = Input(shape=(node_num, ))
encoded = noise.GaussianNoise(0.2)(x)
for i in range(hidden_layers):
encoded = Dense(hidden_neurons[i])(encoded)
encoded = LeakyReLU(0.2)(encoded)
BatchNormalization()
encoded = noise.GaussianNoise(0.2)(encoded)
#add softsign as the last layer
#encoded = Dense(hidden_neurons[-1], activation="softsign")(encoded)
return Model(inputs=x, outputs=encoded)
def sdae(self, PPMI, hidden_neurons):
#local import
from keras.layers import Input, Dense, noise
from keras.models import Model
#Input layer. Corrupt with Gaussian Noise.
inp = Input(shape=(PPMI.shape[1],))
enc = noise.GaussianNoise(0.2)(inp)
#Encoding layers. Last layer is the bottle neck
for neurons in hidden_neurons:
enc = Dense(neurons, activation = 'relu')(enc)
#Decoding layers
dec = Dense(hidden_neurons[-2], activation = 'relu')(enc)
for neurons in hidden_neurons[:-3][::-1]:
dec = Dense(neurons, activation = 'relu')(dec)
dec = Dense(PPMI.shape[1], activation='relu')(dec)
#Train
auto_enc = Model(inputs=inp, outputs=dec)
auto_enc.compile(optimizer='adam', loss='mse')
auto_enc.fit(x=PPMI, y=PPMI, batch_size=10, epochs=5)
encoder = Model(inputs=inp, outputs=enc)
encoder.compile(optimizer='adam', loss='mse')
embeddings = encoder.predict(PPMI)
return embeddings
def get_only_P300_model_LSTM_CNN(eeg_sample_shape):
from keras.regularizers import l2
digit_input = Input(shape=eeg_sample_shape)
# x = Flatten(input_shape=eeg_sample_shape)(digit_input)
from keras.layers.core import Reshape
x = noise.GaussianNoise(sigma=0.0)(digit_input)
x = Reshape((1, eeg_sample_shape[0], eeg_sample_shape[1]))(x)
x = Convolution2D(nb_filter=10,
nb_col=eeg_sample_shape[1],
nb_row=1,
border_mode='valid',
init='glorot_uniform')(x)
x = Activation('tanh')(x)
# result shape (10,25,1)
x = Permute((3, 2, 1))(x)
x = Reshape((eeg_sample_shape[0], 10))(x)
# x = LSTM(10,return_sequences=True, consume_less='mem')(x)
x = LSTM(30, return_sequences=False, consume_less='mem')(x)
# x = LSTM(10, return_sequences=False, consume_less='mem')(x)
# x = LSTM(100, return_sequences=False, consume_less='mem')(x)
# x = Dense(40,activation='tanh')(x)
x = Dense(1)(x)
out = Activation('sigmoid')(x)
model = Model(digit_input, out)
model.summary()
return model
def minst_attention(inc_noise=False, attention=True):
#make layers
inputs = Input(shape=(1,image_size,image_size),name='input')
conv_1a = Convolution2D(32, 3, 3,activation='relu',name='conv_1')
maxp_1a = MaxPooling2D((3, 3), strides=(2,2),name='convmax_1')
norm_1a = crosschannelnormalization(name="convpool_1")
zero_1a = ZeroPadding2D((2,2),name='convzero_1')
conv_2a = Convolution2D(32,3,3,activation='relu',name='conv_2')
maxp_2a = MaxPooling2D((3, 3), strides=(2,2),name='convmax_2')
norm_2a = crosschannelnormalization(name="convpool_2")
zero_2a = ZeroPadding2D((2,2),name='convzero_2')
dense_1a = Lambda(global_average_pooling,output_shape=global_average_pooling_shape,name='dense_1')
dense_2a = Dense(10, activation = 'softmax', init='uniform',name='dense_2')
#make actual model
if inc_noise:
inputs_noise = noise.GaussianNoise(2.5)(inputs)
input_pad = ZeroPadding2D((1,1),input_shape=(1,image_size,image_size),name='input_pad')(inputs_noise)
else:
input_pad = ZeroPadding2D((1,1),input_shape=(1,image_size,image_size),name='input_pad')(inputs)
conv_1 = conv_1a(input_pad)
conv_1 = maxp_1a(conv_1)
conv_1 = norm_1a(conv_1)
conv_1 = zero_1a(conv_1)
conv_2_x = conv_2a(conv_1)
conv_2 = maxp_2a(conv_2_x)
conv_2 = norm_2a(conv_2)
conv_2 = zero_2a(conv_2)
conv_2 = Dropout(0.5)(conv_2)
dense_1 = dense_1a(conv_2)
dense_2 = dense_2a(dense_1)
conv_shape1 = Lambda(change_shape1,output_shape=(32,),name='chg_shape')(conv_2_x)
find_att = dense_2a(conv_shape1)
if attention:
find_att = Lambda(attention_control,output_shape=att_shape,name='att_con')([find_att,dense_2])
else:
find_att = Lambda(no_attention_control,output_shape=att_shape,name='att_con')([find_att,dense_2])
zero_3a = ZeroPadding2D((1,1),name='convzero_3')(find_att)
apply_attention = Merge(mode='mul',name='attend')([zero_3a,conv_1])
conv_3 = conv_2a(apply_attention)
conv_3 = maxp_2a(conv_3)
conv_3 = norm_2a(conv_3)
conv_3 = zero_2a(conv_3)
dense_3 = dense_1a(conv_3)
dense_4 = dense_2a(dense_3)
model = Model(input=inputs,output=dense_4)
return model
def get_only_P300_model_LSTM(eeg_sample_shape):
from keras.regularizers import l2
digit_input = Input(shape=eeg_sample_shape)
# x = Flatten(input_shape=eeg_sample_shape)(digit_input)
x = noise.GaussianNoise(sigma=0.0)(digit_input)
x = LSTM(100, input_shape=eeg_sample_shape, return_sequences=False)(x)
out = Dense(1, activation='sigmoid')(x)
model = Model(digit_input, out)
return model
def new_encdecs(self, compile=True, use_dropout=None, use_noise=None):
self.enc_decs = []
if not use_dropout is None:
self.enc_use_drop = self.drop_rate > 0 and use_dropout
if not use_noise is None:
self.enc_use_noise = self.sigma_base > 0 and use_noise
if self.l1 != 0 or self.l2 != 0:
regularizer = WeightRegularizer(l1=self.l1, l2=self.l2)
else:
regularizer = None
for (i, (n_in, n_out)) in enumerate(
zip(self.layer_sizes[:-1], self.layer_sizes[1:])):
ae = Sequential()
enc_l = []
if self.enc_use_noise:
enc_l.append(
noise.GaussianNoise(self.sigma_base *
(self.sigma_fact**-i),
input_shape=(n_in, )))
enc_l.append(
core.Dense(input_dim=n_in,
output_dim=n_out,
activation='sigmoid',
W_regularizer=regularizer))
if self.enc_use_drop:
enc_l.append(core.Dropout(self.drop_rate))
enc = containers.Sequential(enc_l)
dec = containers.Sequential([
core.Dense(input_dim=n_out,
output_dim=n_in,
activation='sigmoid')
])
ae.add(
core.AutoEncoder(encoder=enc,
decoder=dec,
output_reconstruction=True))
if compile:
ae.compile(loss='mse', optimizer=self.enc_opt)
self.enc_decs.append(ae)
def sda_training(features, labels):
encoder_dims = [1600, 1024, 768]
stacked_encoder = []
int_labels = label_to_category(labels=labels, type='training')
X_train, X_test, y_train, y_test = train_test_split(features, labels, test_size=0.2, random_state=10)
y_train = to_categorical([int_labels[i] for i in y_train])
y_test = to_categorical([int_labels[i] for i in y_test])
for encoder_dim in encoder_dims:
input_dim = X_train.shape[1]
input_img = Input(shape=(input_dim,))
n_layer = noise.GaussianNoise(0.3)(input_img)
encode = Dense(encoder_dim, activation='sigmoid')(n_layer)
decode = Dense(input_dim, activation='sigmoid')(encode)
ae = Model(input_img, decode)
ae.compile(optimizer='adam', loss='mape')
ae.fit(X_train, X_train, epochs=10, batch_size=32, shuffle=True, validation_data=(X_test, X_test))
encoder = Model(input_img, encode)
X_train = encoder.predict(X_train)
X_test = encoder.predict(X_test)
stacked_encoder.append(encoder.layers[-1])
def c3d_localization_model(dropout_probability=0.3, nb_output_type=3):
model = Sequential()
model.add(
Convolution3D(64,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv1',
subsample=(1, 1, 1),
batch_input_shape=(1, 3, 16, 112, 112),
trainable=True))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
border_mode='valid',
name='pool1'))
# 2nd layer group
model.add(
Convolution3D(128,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv2',
subsample=(1, 1, 1),
trainable=True))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool2'))
# 3rd layer group
model.add(
Convolution3D(256,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv3a',
subsample=(1, 1, 1),
trainable=True))
model.add(
Convolution3D(256,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv3b',
subsample=(1, 1, 1),
trainable=True))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool3'))
# 4th layer group
model.add(
Convolution3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv4a',
subsample=(1, 1, 1),
trainable=True))
model.add(
Convolution3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv4b',
subsample=(1, 1, 1),
trainable=True))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool4'))
# 5th layer group
model.add(
Convolution3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv5a',
subsample=(1, 1, 1),
trainable=True))
model.add(
Convolution3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv5b',
subsample=(1, 1, 1),
trainable=True))
model.add(ZeroPadding3D(padding=(0, 1, 1), name='zeropadding'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool5'))
model.add(Flatten(name='flatten'))
#C3D FC layers - wanted
model.add(Dense(4096, activation='relu', name='f6', trainable=True))
#C3D FC layers - unwanted
model.add(Dropout(.5, name='do1'))
model.add(Dense(4096, activation='relu', name='fc7'))
model.add(Dropout(.5, name='do2'))
model.add(Dense(487, activation='softmax', name='fc8'))
# Load weights
if os.path.exists(r"Network\\activity_net\\converted.h5"):
model.load_weights(r"Network\\activity_net\\converted.h5")
else:
model.load_weights(r"Network\\video0\\converted.h5")
# model.load_weights(r"converted.h5")
# Pop C3D FC layers
for _ in range(4):
model.pop()
#reshape C3D output
model.add(Reshape((1, 4096)))
#input of C3D
inp = model.input
print(model.output_shape)
# FC layers group
input_normalized = BatchNormalization(name='normalization')(
model.layers[-1].output)
input_dropout = Dropout(rate=dropout_probability)(input_normalized)
input_noised = noise.GaussianNoise(0.4)(input_dropout)
lstms_inputs = [input_noised]
#input_normalized = BatchNormalization(name='normalization')(input_features)
#input_dropout = Dropout(rate=dropout_probability)(input_normalized)
#input_noised = noise.GaussianNoise(0.4)(input_dropout)
#lstms_inputs = [input_noised]
#CHANGE hyperparameters in the for loop below:
for i in range(1):
previous_layer = lstms_inputs[-1]
lstm = LSTM(150,
batch_input_shape=(1, 1, 4096),
return_sequences=True,
stateful=True,
name='lsmt{}'.format(i + 1))(previous_layer)
lstms_inputs.append(lstm)
output_dropout = Dropout(rate=dropout_probability)(lstms_inputs[-1])
output_noised = noise.GaussianNoise(0.4)(output_dropout)
#CHANGE first input of Dense to nb_classes
output = TimeDistributed(Dense(nb_output_type, activation='sigmoid'),
name='fc')(output_noised)
model = Model(inputs=inp, outputs=output)
return model
def run(model,setting, X_train, Y_train, X_test, X_pca_train, Y_pca_train, X_pca_test):
nb_filters = setting.nb_filter
batch_size = setting.batch_size
nb_epoch = setting.nb_epoch
channels = X_train.shape[1]
bins = X_train.shape[2]
bins_pca = X_pca_train.shape[2]
steps = X_train.shape[3]
X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1] * X_train.shape[2], X_train.shape[3])
X_test = X_test.reshape(X_test.shape[0], 1, X_test.shape[1] * X_test.shape[2], X_test.shape[3])
X_pca_train = X_pca_train.reshape(X_pca_train.shape[0], 1, X_pca_train.shape[1] * X_pca_train.shape[2], X_pca_train.shape[3])
X_pca_test = X_pca_test.reshape(X_pca_test.shape[0], 1, X_pca_test.shape[1] * X_pca_test.shape[2], X_pca_test.shape[3])
Y_train = np_utils.to_categorical(Y_train, 2)
if model is None:
input1 = Input(name="input1", shape = (1, channels * bins, steps))
seq1 = noise.GaussianNoise(setting.noise, input_shape=(1, channels * bins, steps))(input1)
seq1 = Convolution2D(nb_filters, channels * bins, 1,
#init="uniform",
W_regularizer=l2(l=setting.l2),
input_shape=(1, channels * bins, steps),
activation="relu"
)(seq1)
#seq1 = Dropout(0.1)(seq1)
seq1 = Convolution2D(nb_filters, 1, 3,
#init="uniform",
W_regularizer=l2(l=setting.l2),
activation="relu"
)(seq1)
#seq1 = Dropout(0.1)(seq1)
#seq1 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq1)
#seq1 = Dropout(0.1)(seq1)
#seq1 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq1)
#seq1 = Dropout(0.1)(seq1)
#seq1 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq1)
seq1 = Flatten()(seq1)
output1 = Dense(setting.output1, activation="tanh")(seq1)
input2 = Input(name="input2", shape=(1, channels * bins_pca, steps))
seq2 = noise.GaussianNoise(setting.noise, input_shape=(1, channels * bins_pca, steps))(input2)
seq2 = Convolution2D(nb_filters, channels * bins_pca, 1,
#init="uniform",
W_regularizer=l2(l=setting.l2),
input_shape=(1, channels * bins_pca, steps),
activation="relu"
)(seq2)
#seq2 = Dropout(0.1)(seq2)
seq2 = Convolution2D(nb_filters, 1, 3,
#init="uniform",
W_regularizer=l2(l=setting.l2),
activation="relu"
)(seq2)
#seq2 = Dropout(0.1)(seq2)
#seq2 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq2)
#seq2 = Dropout(0.1)(seq2)
#seq2 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq2)
#seq2 = Dropout(0.1)(seq2)
#seq2 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq2)
seq2 = Flatten()(seq2)
output2 = Dense(setting.output2, activation="tanh")(seq2)
merged = merge([output1, output2], mode="concat")
merged = Dense(512, activation="tanh")(merged)
merged = Dense(256, activation="tanh")(merged)
#merged = Dense(128, activation="tanh")(merged)
output = Dense(2, activation="softmax", name="output")(merged)
model = Model(input = [input1, input2], output = [output])
sgd = SGD(lr = 0.01)
model.compile(loss="binary_crossentropy", optimizer = sgd)
model.load_weights("my_model_weights.h5")
history = model.fit({'input1':X_train, 'input2':X_pca_train}, {'output':Y_train}, nb_epoch=nb_epoch, verbose = 1, batch_size = batch_size, class_weight=[1, 1])
predictions = model.predict({'input1':X_test, 'input2':X_pca_test})
output = predictions[:,1]
outputList = []
for i in xrange(output.shape[0]):
if output[i] >= 0.5:
outputList.append(1)
else:
outputList.append(0)
output = numpy.array(outputList)
return output, model
def model_main(
nc: int,
nf_l: int,
nt_l: int,
nf_h: int,
nt_h: int,
*,
mw=False,
preload_weights: str = None,
opti="adam",
noise=False,
noise_amp=0.2,
) -> krm.Model:
if mw:
pre_shape: ty.Tuple[int, ...] = (3, )
else:
pre_shape = ()
i_x_l = klc.Input(shape=pre_shape + (nc, nf_l, nt_l), name="input.lo")
i_x_h = klc.Input(shape=pre_shape + (nc, nf_h, nt_h), name="input.hi")
conv_stack_h = [
kcv.Conv2D(8, (3, 3), name="conv_h.0.0", activation="elu"),
kcv.Conv2D(8, (3, 3), name="conv_h.0.1", activation="elu"),
kcv.MaxPooling2D((2, 2)),
kcv.Conv2D(8, (3, 3), name="conv_h.1.0", activation="elu"),
kcv.Conv2D(8, (3, 3), name="conv_h.1.1", activation="elu"),
kcv.MaxPooling2D((2, 2)),
klc.Flatten(name="conv_h.flatten"),
]
conv_stack_l = [
kcv.Conv2D(8, (3, 3), name="conv_l.0.0", activation="elu"),
kcv.Conv2D(8, (3, 3), name="conv_l.0.1", activation="elu"),
kcv.MaxPooling2D((2, 2)),
klc.Flatten(name="conv_l.flatten"),
]
if mw:
conv_stack_h = [
klc.Permute((1, 4, 2, 3)),
klc.Reshape((-1, nc, nf_h), input_shape=(3, nc, nf_h, nt_h)),
klc.Permute((2, 1, 3)),
] + conv_stack_h
conv_stack_l = [
klc.Permute((1, 4, 2, 3)),
klc.Reshape((-1, nc, nf_l), input_shape=(3, nc, nf_l, nt_l)),
klc.Permute((2, 1, 3)),
] + conv_stack_l
if noise:
conv_stack_h = [kno.GaussianNoise(noise_amp, name="inoise_h")
] + conv_stack_h
conv_stack_l = [kno.GaussianNoise(noise_amp, name="inoise_l")
] + conv_stack_l
conv_l = i_x_l
for layer in conv_stack_l:
conv_l = layer(conv_l)
conv_h = i_x_h
for layer in conv_stack_h:
conv_h = layer(conv_h)
dn_suff = ".mw" if mw else ""
merged_conv = concatenate([conv_l, conv_h])
dense_stack = [
klc.Dense(24, name="dense.0" + dn_suff, activation="elu"),
klc.Dropout(0.5),
klc.Dense(24, name="dense.1" + dn_suff, activation="elu"),
]
y = merged_conv
for layer in dense_stack:
y = layer(y)
y = klc.Dense(6, name="y" + dn_suff, activation="softmax")(y)
m = krm.Model(inputs=[i_x_l, i_x_h], outputs=y)
m.compile(optimizer=opti, loss="categorical_crossentropy")
if preload_weights and mw:
m.load_weights(preload_weights, by_name=True)
return m
def run(setting):
nb_filters = setting.nb_filter
batch_size = setting.batch_size
nb_epoch = setting.nb_epoch
featureName = setting.name
feature = Feature(featureName, "kaggleSolution/kaggleSettings.yml")
X_train, Y_train = feature.loadFromDisk("PCA", "train")
X_train, Y_train = feature.overlapInEachHour(shuffle=True)
X_train, _ = feature.scaleAcrossTime(X_train)
X_test, Y_test = feature.loadFromDisk("PCA", "test")
X_test, _ = feature.scaleAcrossTime(X_test)
channels = X_train.shape[1]
bins = X_train.shape[2]
steps = X_train.shape[3]
X_train = X_train.reshape(X_train.shape[0], 1,
X_train.shape[1] * X_train.shape[2],
X_train.shape[3])
X_test = X_test.reshape(X_test.shape[0], 1,
X_test.shape[1] * X_test.shape[2], X_test.shape[3])
Y_train = np_utils.to_categorical(Y_train, 2)
model = Sequential()
seq1 = noise.GaussianNoise(setting.noise,
input_shape=(1, channels * bins, steps))
seq2 = Convolution2D(
nb_filters,
channels * bins,
1,
#init="uniform",
W_regularizer=l2(l=setting.l2),
input_shape=(1, channels * bins, steps),
activation="relu")
seq3 = Dropout(setting.dropout)
seq4 = Convolution2D(
nb_filters,
1,
3,
#init="uniform",
W_regularizer=l2(l=setting.l2),
activation="relu")
seq5 = Dropout(setting.dropout)
seq6 = Convolution2D(
nb_filters,
1,
3,
#init="uniform",
W_regularizer=l2(l=setting.l2),
activation="relu")
seq7 = Dropout(setting.dropout)
seq8 = Convolution2D(
nb_filters,
1,
3,
#init="uniform",
W_regularizer=l2(l=setting.l2),
activation="relu")
seq9 = Dropout(setting.dropout)
seq10 = Convolution2D(
nb_filters,
1,
3,
#init="uniform",
W_regularizer=l2(l=setting.l2),
activation="relu")
seq11 = Flatten()
seq12 = Dense(setting.output1, activation="tanh")
seq13 = Dense(512, activation="tanh")
seq14 = Dense(256, activation="tanh")
seq15 = Dense(128, activation="tanh")
seq16 = Dense(2, activation="softmax", name="output")
model.add(seq1)
model.add(seq2)
model.add(seq3)
model.add(seq4)
model.add(seq5)
model.add(seq6)
model.add(seq7)
model.add(seq8)
model.add(seq9)
model.add(seq10)
model.add(seq11)
model.add(seq12)
model.add(seq13)
model.add(seq14)
model.add(seq15)
model.add(seq16)
plot(model, to_file=featureName + ".png", show_shapes=True)
sgd = SGD(lr=0.01)
model.compile(loss='binary_crossentropy', optimizer=sgd)
history = model.fit(X_train,
Y_train,
nb_epoch=nb_epoch,
verbose=1,
batch_size=batch_size)
predictions = model.predict(X_test)
output = predictions[:, 1]
output = output.tolist()
ans = zip(Y_test, output)
dataFrame = DataFrame(data=ans, columns=["clip", "preictal"])
dataFrame.to_csv(setting.savePath + featureName + ".csv",
index=False,
header=True)
def model(data, output_file, validation_split=0.9):
#load data from xx into hidde
md = open('neorons.txt', 'r')
line = md.readline()
print line
cnmd = open('layer.txt', 'r')
l = cnmd.readline()
cnm = arg_as_list(line)
hidden_neurons = cnm
hidden_layers = int(l)
train_n = int(validation_split * len(data))
batch_size = 50
e = 50
m = data.shape[1]
if hidden_neurons[0] == 0:
train_data = data
val_data = data
input_sh = Input(shape=(data.shape[1], ))
encoded = noise.GaussianNoise(0.2)(input_sh)
elif hidden_neurons[0] == 1:
train_data = data[:train_n, :]
val_data = data[train_n:, :]
'''train_data = np.expand_dims(train_data, axis=0)
val_data = np.expand_dims(val_data, axis=0)'''
#train_data=tf.reshape(train_data,(train_n,data.shape[1],1))
input_sh = Input(shape=(data.shape[1], ))
input_sh1 = Reshape((data.shape[1], 1))(input_sh)
encoded = noise.GaussianNoise(0.2)(input_sh1)
else:
train_data = data[:train_n, :]
val_data = data[train_n:, :]
input_sh = Input(shape=(data.shape[1], ))
encoded = noise.GaussianNoise(0.2)(input_sh)
print 'ghhhhhhhhhhhhhhh ', encoded.ndim
print hidden_neurons
print 'layer ', hidden_layers
if hidden_neurons[0] == 0:
batch_size = len(data)
encoded = GraphConvolution(
input_dim=data.shape[1],
output_dim=hidden_neurons[1],
support=input_sh,
act=tf.nn.relu,
)(encoded)
elif hidden_neurons[0] == 1:
encoded = Conv1D(filters=1,
kernel_size=178 - hidden_neurons[1],
activation='relu')(encoded)
encoded = Flatten()(encoded)
print 'ghhhhhhhhhhhhhhh ', encoded.ndim
elif hidden_neurons[0] == 2:
encoded = Dense(hidden_neurons[1], activation='relu')(encoded)
encoded = noise.GaussianNoise(0.2)(encoded)
for i in range(2, hidden_layers):
print i, hidden_neurons[i]
print 'ghhhhhhhhhhhhhhh dense ', encoded.ndim
encoded = Dense(hidden_neurons[i], activation='relu')(encoded)
encoded = noise.GaussianNoise(0.2)(encoded)
decoded = Dense(hidden_neurons[-2], activation='relu')(encoded)
print hidden_neurons[-2]
for j in range(hidden_layers - 3, 0, -1):
print 'jhjjjjj ', j, hidden_neurons[j]
decoded = Dense(hidden_neurons[j], activation='relu')(decoded)
print data.shape[1]
decoded = Dense(data.shape[1], activation='sigmoid')(decoded)
autoencoder = Model(inputs=input_sh, outputs=decoded) #bp to train weights
autoencoder.compile(optimizer='adadelta', loss='mse')
#checkpointer = ModelCheckpoint(filepath='bestmodel' + output_file + ".hdf5", verbose=1, save_best_only=True)
earlystopper = EarlyStopping(monitor='val_loss', patience=15, verbose=1)
if hidden_neurons[0] == 1:
train_data1 = data[:train_n, :]
val_data1 = data[train_n:, :]
train_generator = DataGenerator(batch_size)
print 'dfffffffffffffff', train_data.shape
print 'dfffffffffffffff', train_data1.shape
train_generator.fit(train_data, train_data)
val_generator = DataGenerator(batch_size)
val_generator.fit(val_data, val_data)
else:
train_generator = DataGenerator(batch_size)
train_generator.fit(train_data, train_data)
val_generator = DataGenerator(batch_size)
val_generator.fit(val_data, val_data)
h = autoencoder.fit_generator(
train_generator,
steps_per_epoch=len(data) / batch_size,
epochs=e,
validation_data=val_generator,
validation_steps=len(data),
callbacks=[earlystopper],
)
#print 'jkjjkjkkjjk ', h.history
avge = h.history['val_loss'][e - 1]
print 'avge ', avge
enco = Model(inputs=input_sh,
outputs=encoded) #just select the first embedding part
enco.compile(optimizer='adadelta', loss='mse') #configuration ?
reprsn = enco.predict(data, batch_size=batch_size)
return reprsn, avge
# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
print(X_train.shape)
print('Build model...')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
# note: in a situation where your input sequences have a variable length,
# use input_shape=(None, nb_feature).
model.add(
RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(words)), return_sequences=True))
model.add(RNN(HIDDEN_SIZE))
model.add(noise.GaussianNoise(0.5, input_shape=(MAXLEN, len(words))))
# For the decoder's input, we repeat the encoded input for each time step
model.add(RepeatVector(MAXLEN))
# The decoder RNN could be multiple layers stacked or a single layer
for _ in range(LAYERS):
model.add(RNN(HIDDEN_SIZE, return_sequences=True))
# For each of step of the output sequence, decide which character should be chosen
model.add(TimeDistributedDense(len(words)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
import os
if os.path.exists(MODEL_FILE_NAME):
klc.Dropout(0.25, name='enc_do0'),
klc.Dense(1024, name='enc_d1', activation='relu'),
klc.Dropout(0.25, name='enc_do1'),
klc.Dense(H_SIZE, name='enc_d3_y', activation='linear')]
# GENERATOR
gen_ls =\
[klc.Dense(1024, name='gen_d0', activation='relu'),
klc.Dropout(0.25, name='gen_do0'),
klc.Dense(1024, name='gen_d1', activation='relu'),
klc.Dropout(0.25, name='gen_do1'),
klc.Dense(784, name='gen_d3_y', activation='linear')]
# DECODER_HIDDEN
dec_h_ls =\
[kln.GaussianNoise(0.1),
klc.Dropout(0.25, name='dec_h_do0'),
klc.Dense(100, name='dec_h_d1', activation='relu'),
klc.Dropout(0.25, name='dec_h_do2'),
klc.Dense(100, name='dec_h_d2', activation='relu'),
klc.Dense(2, name='dec_d3_h_y', activation='softmax')]
# MODELS
# AE
#
ae_out = i_enc
for l in enc_ls + gen_ls:
ae_out = l(ae_out)
model.add(Dense(512, input_shape=(audio_input_train.shape[1], )))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dense(128))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dense(64))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dense(32, kernel_regularizer=l1_l2(.001)))
model.add(BatchNormalization())
model.add(Activation('sigmoid'))
model.add(noise.GaussianNoise(.05))
model.add(Dense(64))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dense(128))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dense(audio_input_train.shape[1]))
model.compile(loss=corr2_mse_loss, optimizer=adam)
model.summary()
num_iter = 40
loss_history = np.empty((num_iter, 2), dtype='float32')
def run(setting, X_train,X_pca_train, Y_train, X_train_chb01, X_pca_train_chb01, Y_train_chb01):
nb_filters = setting.nb_filter
batch_size = setting.batch_size
nb_epoch = setting.nb_epoch
channels = X_train.shape[1]
bins = X_train.shape[2]
bins_pca = X_pca_train.shape[2]
steps = X_train.shape[3]
X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1] * X_train.shape[2], X_train.shape[3])
X_pca_train = X_pca_train.reshape(X_pca_train.shape[0], 1, X_pca_train.shape[1] * X_pca_train.shape[2], X_pca_train.shape[3])
Y_train = np_utils.to_categorical(Y_train, 2)
X_train_chb01 = X_train_chb01.reshape(X_train_chb01.shape[0], 1, X_train_chb01.shape[1] * X_train_chb01.shape[2], X_train_chb01.shape[3])
X_pca_train_chb01 = X_pca_train_chb01.reshape(X_pca_train_chb01.shape[0], 1, X_pca_train_chb01.shape[1] * X_pca_train_chb01.shape[2], X_pca_train_chb01.shape[3])
Y_train_chb01 = np_utils.to_categorical(Y_train_chb01, 2)
input1 = Input(name="input1", shape = (1, channels * bins, steps))
seq1 = noise.GaussianNoise(setting.noise, input_shape=(1, channels * bins, steps))(input1)
seq1 = Convolution2D(nb_filters, channels * bins, 1,
#init="uniform",
W_regularizer=l2(l=setting.l2),
input_shape=(1, channels * bins, steps),
activation="relu"
)(seq1)
#seq1 = Dropout(0.2)(seq1)
seq1 = Convolution2D(nb_filters, 1, 3,
#init="uniform",
W_regularizer=l2(l=setting.l2),
activation="relu"
)(seq1)
#seq1 = Dropout(0.2)(seq1)
#seq1 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq1)
#seq1 = Dropout(0.2)(seq1)
#seq1 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq1)
#seq1 = Dropout(0.2)(seq1)
#seq1 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq1)
seq1 = Flatten()(seq1)
output1 = Dense(setting.output1, activation="tanh")(seq1)
input2 = Input(name="input2", shape=(1, channels * bins_pca, steps))
seq2 = noise.GaussianNoise(setting.noise, input_shape=(1, channels * bins_pca, steps))(input2)
seq2 = Convolution2D(nb_filters, channels * bins_pca, 1,
#init="uniform",
W_regularizer=l2(l=setting.l2),
input_shape=(1, channels * bins_pca, steps),
activation="relu"
)(seq2)
#seq2 = Dropout(0.2)(seq2)
seq2 = Convolution2D(nb_filters, 1, 3,
#init="uniform",
W_regularizer=l2(l=setting.l2),
activation="relu"
)(seq2)
#seq2 = Dropout(0.2)(seq2)
#seq2 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq2)
#seq2 = Dropout(0.2)(seq2)
#seq2 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq2)
#seq2 = Dropout(0.2)(seq2)
#seq2 = Convolution2D(nb_filters, 1, 3,
# #init="uniform",
# W_regularizer=l2(l=setting.l2),
# activation="relu"
# )(seq2)
seq2 = Flatten()(seq2)
output2 = Dense(setting.output2, activation="tanh")(seq2)
merged = merge([output1, output2], mode="concat")
merged = Dense(512, activation="tanh")(merged)
merged = Dense(256, activation="tanh")(merged)
#merged = Dense(128, activation="tanh")(merged)
output = Dense(2, activation="softmax", name="output")(merged)
model = Model(input = [input1, input2], output = [output])
sgd = SGD(lr = 0.01)
model.compile(loss="binary_crossentropy", optimizer = sgd, metrics=['accuracy'])
#callback = ModelCheckpoint(filepath = "my_model_weights.h5", save_best_only = True)
#history = model.fit({'input1':X_train, 'input2':X_pca_train}, {'output':Y_train}, callbacks = [callback], validation_data = ({"input1":X_train_chb01, "input2":X_pca_train_chb01}, {"output":Y_train_chb01}), nb_epoch=nb_epoch, verbose = 1, batch_size = batch_size, class_weight = [1, 1])
history = model.fit({'input1':X_train, 'input2':X_pca_train}, {'output':Y_train}, validation_split = 0.3, batch_size = batch_size, class_weight = [1, 1], nb_epoch = nb_epoch)
model.save_weights("my_model_weights.h5")
class LayerCorrectnessTest(keras_parameterized.TestCase):
def setUp(self):
super(LayerCorrectnessTest, self).setUp()
# Set two virtual CPUs to test MirroredStrategy with multiple devices
cpus = tf.config.list_physical_devices('CPU')
tf.config.set_logical_device_configuration(cpus[0], [
tf.config.LogicalDeviceConfiguration(),
tf.config.LogicalDeviceConfiguration(),
])
def _create_model_from_layer(self, layer, input_shapes):
inputs = [layers.Input(batch_input_shape=s) for s in input_shapes]
if len(inputs) == 1:
inputs = inputs[0]
y = layer(inputs)
model = models.Model(inputs, y)
model.compile('sgd', 'mse')
return model
@parameterized.named_parameters(
('LeakyReLU', advanced_activations.LeakyReLU, (2, 2)),
('PReLU', advanced_activations.PReLU, (2, 2)),
('ELU', advanced_activations.ELU, (2, 2)),
('ThresholdedReLU', advanced_activations.ThresholdedReLU, (2, 2)),
('Softmax', advanced_activations.Softmax, (2, 2)),
('ReLU', advanced_activations.ReLU, (2, 2)),
('Conv1D', lambda: convolutional.Conv1D(2, 2), (2, 2, 1)),
('Conv2D', lambda: convolutional.Conv2D(2, 2), (2, 2, 2, 1)),
('Conv3D', lambda: convolutional.Conv3D(2, 2), (2, 2, 2, 2, 1)),
('Conv2DTranspose', lambda: convolutional.Conv2DTranspose(2, 2),
(2, 2, 2, 2)),
('SeparableConv2D', lambda: convolutional.SeparableConv2D(2, 2),
(2, 2, 2, 1)),
('DepthwiseConv2D', lambda: convolutional.DepthwiseConv2D(2, 2),
(2, 2, 2, 1)),
('UpSampling2D', convolutional.UpSampling2D, (2, 2, 2, 1)),
('ZeroPadding2D', convolutional.ZeroPadding2D, (2, 2, 2, 1)),
('Cropping2D', convolutional.Cropping2D, (2, 3, 3, 1)),
('ConvLSTM2D',
lambda: convolutional_recurrent.ConvLSTM2D(4, kernel_size=(2, 2)),
(4, 4, 4, 4, 4)),
('Dense', lambda: core.Dense(2), (2, 2)),
('Dropout', lambda: core.Dropout(0.5), (2, 2)),
('SpatialDropout2D', lambda: core.SpatialDropout2D(0.5), (2, 2, 2, 2)),
('Activation', lambda: core.Activation('sigmoid'), (2, 2)),
('Reshape', lambda: core.Reshape((1, 4, 1)), (2, 2, 2)),
('Permute', lambda: core.Permute((2, 1)), (2, 2, 2)),
('Attention', dense_attention.Attention, [(2, 2, 3), (2, 3, 3),
(2, 3, 3)]),
('AdditiveAttention', dense_attention.AdditiveAttention, [(2, 2, 3),
(2, 3, 3),
(2, 3, 3)]),
('Embedding', lambda: embeddings.Embedding(4, 4),
(2, 4), 2e-3, 2e-3, np.random.randint(4, size=(2, 4))),
('LocallyConnected1D', lambda: local.LocallyConnected1D(2, 2),
(2, 2, 1)),
('LocallyConnected2D', lambda: local.LocallyConnected2D(2, 2),
(2, 2, 2, 1)),
('Add', merge.Add, [(2, 2), (2, 2)]),
('Subtract', merge.Subtract, [(2, 2), (2, 2)]),
('Multiply', merge.Multiply, [(2, 2), (2, 2)]),
('Average', merge.Average, [(2, 2), (2, 2)]),
('Maximum', merge.Maximum, [(2, 2), (2, 2)]),
('Minimum', merge.Minimum, [(2, 2), (2, 2)]),
('Concatenate', merge.Concatenate, [(2, 2), (2, 2)]),
('Dot', lambda: merge.Dot(1), [(2, 2), (2, 2)]),
('GaussianNoise', lambda: noise.GaussianNoise(0.5), (2, 2)),
('GaussianDropout', lambda: noise.GaussianDropout(0.5), (2, 2)),
('AlphaDropout', lambda: noise.AlphaDropout(0.5), (2, 2)),
('BatchNormalization', normalization_v2.BatchNormalization,
(2, 2), 1e-2, 1e-2),
('LayerNormalization', normalization.LayerNormalization, (2, 2)),
('LayerNormalizationUnfused',
lambda: normalization.LayerNormalization(axis=1), (2, 2, 2)),
('MaxPooling2D', pooling.MaxPooling2D, (2, 2, 2, 1)),
('AveragePooling2D', pooling.AveragePooling2D, (2, 2, 2, 1)),
('GlobalMaxPooling2D', pooling.GlobalMaxPooling2D, (2, 2, 2, 1)),
('GlobalAveragePooling2D', pooling.GlobalAveragePooling2D,
(2, 2, 2, 1)),
('SimpleRNN', lambda: recurrent.SimpleRNN(units=4),
(4, 4, 4), 1e-2, 1e-2),
('GRU', lambda: recurrent.GRU(units=4), (4, 4, 4)),
('LSTM', lambda: recurrent.LSTM(units=4), (4, 4, 4)),
('GRUV2', lambda: recurrent_v2.GRU(units=4), (4, 4, 4)),
('LSTMV2', lambda: recurrent_v2.LSTM(units=4), (4, 4, 4)),
('TimeDistributed', lambda: wrappers.TimeDistributed(core.Dense(2)),
(2, 2, 2)),
('Bidirectional',
lambda: wrappers.Bidirectional(recurrent.SimpleRNN(units=4)),
(2, 2, 2)),
('AttentionLayerCausal',
lambda: dense_attention.Attention(causal=True), [(2, 2, 3), (2, 3, 3),
(2, 3, 3)]),
('AdditiveAttentionLayerCausal',
lambda: dense_attention.AdditiveAttention(causal=True), [(2, 3, 4),
(2, 3, 4),
(2, 3, 4)]),
)
def test_layer(self,
f32_layer_fn,
input_shape,
rtol=2e-3,
atol=2e-3,
input_data=None):
"""Tests a layer by comparing the float32 and mixed precision weights.
A float32 layer, a mixed precision layer, and a distributed mixed precision
layer are run. The three layers are identical other than their dtypes and
distribution strategies. The outputs after predict() and weights after fit()
are asserted to be close.
Args:
f32_layer_fn: A function returning a float32 layer. The other two layers
will automatically be created from this
input_shape: The shape of the input to the layer, including the batch
dimension. Or a list of shapes if the layer takes multiple inputs.
rtol: The relative tolerance to be asserted.
atol: The absolute tolerance to be asserted.
input_data: A Numpy array with the data of the input. If None, input data
will be randomly generated
"""
if f32_layer_fn == convolutional.ZeroPadding2D and \
tf.test.is_built_with_rocm():
return
if isinstance(input_shape[0], int):
input_shapes = [input_shape]
else:
input_shapes = input_shape
strategy = create_mirrored_strategy()
f32_layer = f32_layer_fn()
# Create the layers
assert f32_layer.dtype == f32_layer._compute_dtype == 'float32'
config = f32_layer.get_config()
config['dtype'] = policy.Policy('mixed_float16')
mp_layer = f32_layer.__class__.from_config(config)
distributed_mp_layer = f32_layer.__class__.from_config(config)
# Compute per_replica_input_shapes for the distributed model
global_batch_size = input_shapes[0][0]
assert global_batch_size % strategy.num_replicas_in_sync == 0, (
'The number of replicas, %d, does not divide the global batch size of '
'%d' % (strategy.num_replicas_in_sync, global_batch_size))
per_replica_batch_size = (global_batch_size //
strategy.num_replicas_in_sync)
per_replica_input_shapes = [(per_replica_batch_size, ) + s[1:]
for s in input_shapes]
# Create the models
f32_model = self._create_model_from_layer(f32_layer, input_shapes)
mp_model = self._create_model_from_layer(mp_layer, input_shapes)
with strategy.scope():
distributed_mp_model = self._create_model_from_layer(
distributed_mp_layer, per_replica_input_shapes)
# Set all model weights to the same values
f32_weights = f32_model.get_weights()
mp_model.set_weights(f32_weights)
distributed_mp_model.set_weights(f32_weights)
# Generate input data
if input_data is None:
# Cast inputs to float16 to avoid measuring error from having f16 layers
# cast to float16.
input_data = [
np.random.normal(size=s).astype('float16')
for s in input_shapes
]
if len(input_data) == 1:
input_data = input_data[0]
# Assert all models have close outputs.
f32_output = f32_model.predict(input_data)
mp_output = mp_model.predict(input_data)
self.assertAllClose(mp_output, f32_output, rtol=rtol, atol=atol)
self.assertAllClose(distributed_mp_model.predict(input_data),
f32_output,
rtol=rtol,
atol=atol)
# Run fit() on models
output = np.random.normal(
size=f32_model.outputs[0].shape).astype('float16')
for model in f32_model, mp_model, distributed_mp_model:
model.fit(input_data, output, batch_size=global_batch_size)
# Assert all models have close weights
f32_weights = f32_model.get_weights()
self.assertAllClose(mp_model.get_weights(),
f32_weights,
rtol=rtol,
atol=atol)
self.assertAllClose(distributed_mp_model.get_weights(),
f32_weights,
rtol=rtol,
atol=atol)
def test_GaussianNoise():
layer = noise.GaussianNoise(sigma=1., input_shape=input_shape)
_runner(layer)