def MakeConvNet(Size, batch_size, epochs, optimizer, learning_rate, train_list, validation_list):
input_img = Input(shape=Size)
model = input_img
CurrentInput = InputData
Channels = Size[2] #the input dim at the first layer is 1, since the input image is grayscale
for i in range(len(NumKernels)-1): #number of layers
NumKernel=NumKernels[i]
FilterSize = FilterSizes[i]
print(i)
model = Conv2DTranspose(NumKernel, (FilterSize, FilterSize), padding='same', kernel_initializer='he_normal', use_bias=False)(model)
print(model.shape)
model = BatchNormalization()(model)
print(model.shape)
# model = Activation('relu')(model)
model = LeakyReLU()(model)
print(model.shape)
model = Conv2DTranspose(4, (3, 3), padding='same', kernel_initializer='he_normal', use_bias=False)(model)
print(model.shape)
model = BatchNormalization()(model)
print(model.shape)
# model = Activation('relu')(model)
model = SubpixelConv2D(2)(model)
model = LeakyReLU()(model)
print(model.shape)
# print(input_img.shape)
model = Model(input_img, model)
adam = Adadelta()
sgd = SGD(lr=learning_rate, momentum=0.9, decay=1e-4, nesterov=False, clipnorm=1)
if optimizer == 0:
model.compile(adam, loss='mean_absolute_error', metrics=[ssim, ssim_metric, PSNR])
else:
model.compile(sgd, loss='mean_absolute_error', metrics=[ssim, ssim_metric, PSNR])
model.summary()
mycallback = MyCallback(model)
timestamp = time.strftime("%m%d-%H%M", time.localtime(time.time()))
csv_logger = callbacks.CSVLogger('data/callbacks/training_{}.log'.format(timestamp))
filepath="./checkpoints/weights-{epoch:03d}-{PSNR:.2f}-{ssim:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor=PSNR, verbose=1, mode='max')
callbacks_list = [mycallback, checkpoint, csv_logger]
with open('./model/subcnn_architecture_{}.json'.format(timestamp), 'w') as f:
f.write(model.to_json())
history = model.fit_generator(image_gen(train_list, batch_size=batch_size),
steps_per_epoch=(3600)*len(train_list) // batch_size,
validation_data=image_gen(validation_list,batch_size=batch_size),
validation_steps=(3600)*len(validation_list) // batch_size,
epochs=epochs,
workers=1024,
callbacks=callbacks_list,
verbose=1)
print("Done training!!!")
def compiled_single_model(model_input_shape):
input = Input(shape=model_input_shape)
model = Conv2D(32, (3, 3), activation='relu')(input)
model = BatchNormalization(axis=3)(model)
model = Activation('relu')(model)
model = MaxPooling2D((3, 3))(model)
model = Conv2D(64, (3, 3), activation='relu')(model)
model = BatchNormalization(axis=3)(model)
model = Activation('relu')(model)
model = MaxPooling2D((3, 3))(model)
model = Conv2D(128, (3, 3), activation='relu')(model)
model = BatchNormalization(axis=3)(model)
model = Activation('relu')(model)
model = MaxPooling2D((3, 3))(model)
model = Conv2D(256, (3, 3), activation='relu')(model)
model = AveragePooling2D((7, 7))(model)
model = Flatten()(model)
model = Dense(num_classes, activation=last_activation)(model)
model = Model(inputs=input, outputs=model)
model.compile(
loss=loss,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'],
)
return model
def compiled_single_model(model_input_shape):
input = Input(shape=model_input_shape)
model = Conv2D(32, (3, 3), activation='relu')(input)
model = BatchNormalization(axis=3)(model)
model = Activation('relu')(model)
model = MaxPooling2D((3, 3))(model)
model = Conv2D(64, (3, 3), activation='relu')(model)
model = MaxPooling2D((3, 3))(model)
model = Conv2D(128, (3, 3), activation='relu')(model)
model = BatchNormalization(axis=3)(model)
model = Activation('relu')(model)
model = MaxPooling2D((3, 3))(model)
model = Conv2D(256, (3, 3), activation='relu')(model)
model = MaxPooling2D((3, 3))(model)
model = Dropout(.25)(model)
model = Flatten()(model)
model = Dense(128, activation='relu')(model)
model = Dropout(.25)(model)
model = Dense(64, activation='relu')(model)
model = Dropout(.25)(model)
model = Dense(num_classes, activation=last_activation)(model)
model = Model(inputs=input, outputs=model)
if len(args.gpus) > 1:
model = keras.utils.multi_gpu_model(model,
len(args.gpus),
cpu_merge=False)
model.compile(
loss=loss,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'],
)
return model
def create_model():
input_model = Input(shape=(X.shape[1], X.shape[2]))
model = BatchNormalization()(input_model)
model = LSTM(ndim,
dropout=0.1,
recurrent_dropout=0.2,
return_sequences=True)(model)
model = LSTM(ndim * 2,
dropout=0.1,
recurrent_dropout=0.2,
return_sequences=True)(model)
model = LSTM(ndim,
dropout=0.1,
recurrent_dropout=0.2,
return_sequences=True)(model)
model = Flatten()(model)
model = Dense(n_classes, activation='softmax')(model)
model = Model(input_model, model)
# model compilation
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
return model
res_img = model
from vdsr_deconv import crop
input_img1 = crop(1, 2, -2)(input_img)
input_img1 = crop(2, 2, -2)(input_img1)
output_img = merge.Add()([res_img, input_img1])
model = Model(input_img, output_img)
print(type(model))
adam = Adadelta()
from vdsr_deconv import *
model.load_weights(w_path, by_name=True)
model.compile(adam, loss='mse', metrics=[ssim, ssim_metric, PSNR])
vdsr = model
vdsr.summary()
print(type(vdsr))
li = os.listdir(img_path)
target_path = '%s/%s/' % (img_path, dst_path)
os.makedirs(target_path, exist_ok=True)
for filename in li:
if '.tif' in filename:
img = numpy.array(imread(os.path.join(img_path, filename)))
img = cv2.resize(img, target_size, interpolation=cv2.INTER_CUBIC)
# img = imresize(img, target_size, interp='bicubic')
img = numpy.array(img) / 127.5 - 1.
seed = 42
)
val_set_angle_diff= val_datagen_angle_diff.flow_from_directory(
valdiranglediff,
target_size=(65, 3420),
color_mode='grayscale',
batch_size=32,
class_mode='categorical',
shuffle = False,
seed = 42
)
sgd = optimizers.SGD(lr = 0.01, momentum = 0.9,clipnorm = 1.0)
adam = optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999,clipnorm = 1.0)
angdiffmodel.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
checkpoint1 = ModelCheckpoint('modelangdiff.h5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.7, min_delta = 0.0005,
patience=20, min_lr=0.0001, verbose = 1)
callbacks_list = [checkpoint1,reduce_lr]
H = angdiffmodel.fit_generator(
training_set_angle_diff,
steps_per_epoch=87, #No of train images / batch size
#steps_per_epoch=1,
epochs=1000,
validation_data = val_set_angle_diff,
validation_steps = 87, #No of validation images / batch size
callbacks=callbacks_list)
def convolution_model(num_speakers=2):
# == Audio convolution layers ==
# Input layer for functional model
audio_inputs = Input(shape=(298, 257, 2))
# Convolution layers
model = Conv2D(96, kernel_size=(1,7), padding='same', dilation_rate=(1,1))(audio_inputs)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(7,1), padding='same', dilation_rate=(1,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(1,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(2,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(4,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(8,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(16,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(32,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(1,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(2,2))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(4,4))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(8,8))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(16,16))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(32,32))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(8, kernel_size=(1,1), padding='same', dilation_rate=(1,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
audio_outputs = Reshape((298, 8*257))(model)
audio_model = Model(audio_inputs, audio_outputs)
# == Visual convolution layers ==
visual_inputs_list = []
visual_outputs_list = []
for i in range(num_speakers):
# Input layer for functional model
# TODO: check the correctness of the input layer
visual_inputs = Input(shape=(75, 1, 1024))
# Convolution layers
model = Conv2D(256, kernel_size=(7,1), padding='same', dilation_rate=(1,1))(visual_inputs)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(256, kernel_size=(5,1), padding='same', dilation_rate=(1,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(256, kernel_size=(5,1), padding='same', dilation_rate=(2,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(256, kernel_size=(5,1), padding='same', dilation_rate=(4,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(256, kernel_size=(5,1), padding='same', dilation_rate=(8,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Conv2D(256, kernel_size=(5,1), padding='same', dilation_rate=(16,1))(model)
model = BatchNormalization()(model)
model = Activation("relu")(model)
model = Reshape((75, 256, 1))(model)
# TODO - upsampling code taken from other repo
def UpSampling2DBilinear(size):
return Lambda(lambda x: tf.image.resize_bilinear(x, size, align_corners=True))
model = UpSampling2DBilinear((298, 256))(model)
visual_outputs = Reshape((298, 256))(model)
visual_model = Model(visual_inputs, visual_outputs)
visual_model.summary()
visual_inputs_list.append(visual_inputs)
visual_outputs_list.append(visual_outputs)
# == AV fused neural network ==
fused_model = concatenate([audio_outputs]+visual_outputs_list, axis=2)
# # TEMP
# fused_model = audio_model
# AV fusion step(s)
fused_model = TimeDistributed(Flatten())(fused_model)
# BLSTM
new_matrix_length = 400
# fused_model = Bidirectional(LSTM(new_matrix_length//2, return_sequences=True, input_shape=(298, 257*8)))(fused_model)
fused_model = Bidirectional(LSTM(new_matrix_length//2, return_sequences=True, input_shape=(298, 257*8 + 256*num_speakers)))(fused_model)
# Fully connected layers
fused_model = Dense(600, activation="relu")(fused_model)
fused_model = Dense(600, activation="relu")(fused_model)
fused_model = Dense(600, activation="relu")(fused_model)
# Output layer (i.e. complex masks)
fused_model = Dense(257*2*num_speakers, activation="sigmoid")(fused_model) # TODO: check if this is more correct (based on the paper)
outputs_complex_masks = Reshape((298, 257, 2, num_speakers), name="output_layer")(fused_model)
av_model = Model(inputs=[audio_inputs]+visual_inputs_list, outputs=outputs_complex_masks)
model = av_model
# Print the output shapes of each model layer
for layer in model.layers:
name = layer.get_config()["name"]
if "batch_normal" in name or "activation" in name:
continue
print(layer.output_shape, "\t", name)
# Alternatively, print the default keras model summary
model.summary()
# Compile the model before training
model.compile(optimizer='adam', loss='mse', metrics=['accuracy'])
return model
Mobilenet = BatchNormalization()(Mobilenet) Mobilenet = LeakyReLU()(Mobilenet) Mobilenet = MaxPooling2D(pool_size = (2,2), padding = 'same')(Mobilenet) Mobilenet = identity_block(Mobilenet, 128) Mobilenet = identity_block(Mobilenet, 256) Mobilenet = identity_block(Mobilenet, 256) Mobilenet = Flatten()(Mobilenet) Mobilenet = identity_block(Mobilenet, 64, Con = False) Mobilenet = identity_block(Mobilenet, 64, Con = False) Mobilenet = Dense(units = 7, activation='softmax')(Mobilenet) Mobilenet = Model(input_img, Mobilenet) Mobilenet.compile(optimizer='adamax', loss='categorical_crossentropy', metrics = ['accuracy']) #Image generator from keras.preprocessing.image import ImageDataGenerator img_generator = ImageDataGenerator(rotation_range = 20, width_shift_range = 0.2, height_shift_range = 0.2, zoom_range = 0.3) #callback函數 from keras.callbacks import EarlyStopping, ReduceLROnPlateau lr_adjust = ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience = 4, verbose = 1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) earlystopping = EarlyStopping(monitor='val_acc', patience = 4, verbose = 1, mode='auto') epochs = 1 batch_size = 64 Mobilenet.fit_generator(img_generator.flow(X_train, Y_train, batch_size = batch_size),
model = MaxPooling2D(strides=2, padding='valid')(model)
model = BatchNormalization()(model)
model = Activation('relu')(model)
model = Flatten()(model)
model = Dense(4096, activation='relu')(model)
model = Dense(4096, activation='relu')(model)
outputs = Dense(130, activation='sigmoid')(model)
model = Model(inputs=inputs, outputs=outputs)
sgd = optimizers.SGD(lr=initial_lr,
momentum=0.9,
decay=weight_decay,
nesterov=True)
model.compile(optimizer=sgd, loss='binary_crossentropy', metrics=['acc_exam'])
model.summary()
checkpoint_name = checkpoint_path + '130bestmodel.h5'
#define callbacks
def schedule(epoch):
lr = 1e-3
if 0 < epoch <= 50:
lr = 1e-1
elif 50 < epoch <= 100:
lr = 1e-2
elif 100 < epoch <= 150:
lr = 1e-3
else:
model = Dense(512, activation='relu')(model)
x1 = Dense(6, activation='softmax')(model)
x2 = Dense(11, activation='softmax')(model)
x3 = Dense(11, activation='softmax')(model)
x4 = Dense(11, activation='softmax')(model)
x5 = Dense(11, activation='softmax')(model)
x6 = Dense(11, activation='softmax')(model)
x = [x1, x2, x3, x4, x5, x6]
model = Model(inputs=input_, outputs=x)
model.summary()
model.compile(loss='sparse_categorical_crossentropy',
optimizer="adam",
metrics=['accuracy'])
model.summary()
# Y_train = np_utils.to_categorical(y_train) # One-hot encode the labels
# Y_test = np_utils.to_categorical(y_test) # One-hot encode the labels
fit_data = model.fit(X_train, y_train, batch_size=64, epochs=1, verbose=2)
test_data = model.evaluate(X_test, y_test)
print('Test accuracy: {:.4f}'.format(test_data[1]))
print('Test loss: {:.4f}'.format(test_data[0]))
# model.save('recognizer_model_svhn_v2_adam.h5')
# del model
# model = load_model('recognizer_model_svhn_v2.h5')
merged_layer = Concatenate()([discriminator, labels])
discriminator = Dense(512, activation='relu')(merged_layer)
# Output
discriminator = Dense(1, activation='sigmoid')(discriminator)
discriminator = Model(inputs=[img_input, labels],
outputs=discriminator,
name='discriminator')
# prints a summary representation of your model
discriminator.summary()
# # Optimizer
discriminator.compile(Adam(lr=0.0002, beta_1=0.5),
loss='binary_crossentropy',
metrics=['binary_accuracy'])
# **************************** GAN
discriminator.trainable = False
label = Input(shape=(10, ), name='label')
z = Input(shape=(100, ), name='z')
fake_img = generator([z, label])
validity = discriminator([fake_img, label])
d_g = Model([z, label], validity, name='adversarial')
d_g.compile(Adam(lr=0.0004, beta_1=0.5),
loss='binary_crossentropy',
# end of additions
model = Flatten()(model)
model = Dense(1024, activation="relu")(model)
model = Dropout(rate=0.3)(model)
dense = Dense(512, activation="relu")(model)
head_root = Dense(168, activation='softmax')(dense)
head_vowel = Dense(11, activation='softmax')(dense)
head_consonant = Dense(7, activation='softmax')(dense)
model = Model(inputs=inputs, outputs=[head_root, head_vowel, head_consonant])
#%%
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
#%%
class MultiOutputDataGenerator(keras.preprocessing.image.ImageDataGenerator):
def flow(self,
x,
y=None,
batch_size=32,
shuffle=True,
sample_weight=None,
seed=None,
save_to_dir=None,
save_prefix='',
save_format='png',
activation='relu')(model)
model = BatchNormalization()(model)
model = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='valid')(model)
model = Conv2D(384, (3, 3), strides=(1, 1), padding='same',
activation='relu')(model)
model = Conv2D(384, (3, 3), strides=(1, 1), padding='same',
activation='relu')(model)
model = Conv2D(256, (3, 3), strides=(1, 1), padding='same',
activation='relu')(model)
model = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='valid')(model)
model = Flatten()(model)
model = Dense(1024, activation='relu')(model)
model = Dropout(0.5)(model)
model = Dense(1024, activation='relu')(model)
model = Dropout(0.5)(model)
output = Dense(classes, activation='softmax')(model)
model = Model(inputs=input, outputs=output)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
#tensorboard = TensorBoard(log_dir=r'E:\sansu\python\tensorboard')
#es=EarlyStopping(monitor='val_acc', patience=2, verbose=0,min_delta=0.002,mode='max')
model.fit_generator(train_data,
steps_per_epoch=(train_data.samples // train_batch_size),
epochs=40,
validation_data=test_data,
validation_steps=(test_data.samples // test_batch_size))
