def ConvolutionalNet(vocabulary_size, embedding_dimension, input_length, embedding_weights=None):
model = Sequential()
if embedding_weights is None:
model.add(Embedding(vocabulary_size, embedding_dimension, input_length=input_length, trainable=False))
else:
model.add(Embedding(vocabulary_size, embedding_dimension, input_length=input_length, weights=[embedding_weights], trainable=False))
model.add(Convolution1D(32, 2, kernel_regularizer=l2(0.005)))
model.add(BatchNormalization())
model.add(Activation(activations.relu))
model.add(Convolution1D(32, 2, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation(activations.relu))
model.add(Convolution1D(32, 2, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation(activations.relu))
model.add(MaxPooling1D(17))
model.add(Flatten())
model.add(Dense(1, use_bias=True, kernel_regularizer=l2(0.001)))
model.add(BatchNormalization())
model.add(Activation(activations.sigmoid))
return model
def generate_cnn_model(self) -> Model:
sequence_input = Input(shape=(None, 3000, 1))
# convolutional layer and dropout [1]
sequence = TimeDistributed(
self.__generate_base_model())(sequence_input)
sequence = Convolution1D(filters=128,
kernel_size=self.__kernel_size,
padding=self.__padding_same,
activation=activations.relu)(sequence)
sequence = SpatialDropout1D(rate=self.__dropout_rate)(sequence)
# convolutional layer and dropout [2]
sequence = Convolution1D(filters=128,
kernel_size=self.__kernel_size,
padding=self.__padding_same,
activation=activations.relu)(sequence)
sequence = Dropout(rate=self.__dropout_rate)(sequence)
# last convolution and model generation
model = models.Model(inputs=sequence_input,
outputs=Convolution1D(
filters=self.__classes_number,
kernel_size=self.__kernel_size,
padding=self.__padding_same,
activation=activations.softmax)(sequence))
# compile model
model.compile(optimizer=optimizers.Adam(),
loss=losses.sparse_categorical_crossentropy,
metrics=self.__metrics)
return model
def __init__(self, input_shape, latent_size):
super(SequenceEncoder, self).__init__()
def sampling(args):
z_mean_, z_log_var_ = args
batch_size = shape(z_mean_)[0]
epsilon = random_normal(shape=(batch_size, latent_size),
mean=0.,
stddev=0.01)
return z_mean_ + exp(z_log_var_ / 2) * epsilon
encoder_conv_1 = Convolution1D(512,
7,
activation='relu',
name='encoder_conv_1')
encoder_conv_2 = Convolution1D(256,
5,
activation='relu',
name='encoder_conv_2')
encoder_conv_3 = Convolution1D(128,
3,
activation='relu',
name='encoder_conv_3')
encoder_flatten = Flatten(name='encoder_flatten')
encoder_dense_latent = Dense(latent_size,
activation='relu',
name='encoder_latent')
encoder_z_mean = Dense(latent_size,
activation='linear',
name='encoder_z_mean')
encoder_z_log_var = Dense(latent_size,
activation='linear',
name='encoder_z_log_var')
encoder_z = Lambda(sampling,
output_shape=(latent_size, ),
name='encoder_z')
encoder_input = Input(shape=input_shape, name='encoder_input')
hidden = encoder_conv_1(encoder_input)
hidden = encoder_conv_2(hidden)
hidden = encoder_conv_3(hidden)
hidden = encoder_flatten(hidden)
latent = encoder_dense_latent(hidden)
z_mean = encoder_z_mean(latent)
z_log_var = encoder_z_log_var(latent)
encoder_output = encoder_z([z_mean, z_log_var])
super(SequenceEncoder, self).__init__(encoder_input, encoder_output)
def vae_loss(x, x_mean):
x = flatten(x)
x_mean = flatten(x_mean)
xent_loss = input_shape[0] * binary_crossentropy(x, x_mean)
kl_loss = -0.5 * mean(
1 + z_log_var - square(z_mean) - exp(z_log_var), axis=-1)
return xent_loss + kl_loss
self.__vae_loss = vae_loss
def create_model(self):
K.clear_session()
input0 = Input(shape=(self.c['sentencepad'], self.c['wordvectdim']))
input1 = Input(shape=(self.c['sentencepad'], self.c['wordvectdim']))
Convolt_Layer = []
MaxPool_Layer = []
Flatten_Layer = []
for kernel_size, filters in self.c['cnnfilters'].items():
Convolt_Layer.append(
Convolution1D(filters=filters,
kernel_size=kernel_size,
padding='valid',
activation=self.c['cnnactivate'],
kernel_initializer=self.c['cnninitial']))
MaxPool_Layer.append(
MaxPooling1D(pool_size=int(self.c['sentencepad'] -
kernel_size + 1)))
Flatten_Layer.append(Flatten())
Convolted_tensor0 = []
Convolted_tensor1 = []
for channel in range(len(self.c['cnnfilters'])):
Convolted_tensor0.append(Convolt_Layer[channel](input0))
Convolted_tensor1.append(Convolt_Layer[channel](input1))
MaxPooled_tensor0 = []
MaxPooled_tensor1 = []
for channel in range(len(self.c['cnnfilters'])):
MaxPooled_tensor0.append(MaxPool_Layer[channel](
Convolted_tensor0[channel]))
MaxPooled_tensor1.append(MaxPool_Layer[channel](
Convolted_tensor1[channel]))
Flattened_tensor0 = []
Flattened_tensor1 = []
for channel in range(len(self.c['cnnfilters'])):
Flattened_tensor0.append(Flatten_Layer[channel](
MaxPooled_tensor0[channel]))
Flattened_tensor1.append(Flatten_Layer[channel](
MaxPooled_tensor1[channel]))
if len(self.c['cnnfilters']) > 1:
Flattened_tensor0 = concatenate(Flattened_tensor0)
Flattened_tensor1 = concatenate(Flattened_tensor1)
else:
Flattened_tensor0 = Flattened_tensor0[0]
Flattened_tensor1 = Flattened_tensor1[0]
absDifference = Lambda(lambda X: K.abs(X[0] - X[1]))(
[Flattened_tensor0, Flattened_tensor1])
mulDifference = multiply([Flattened_tensor0, Flattened_tensor1])
allDifference = concatenate([absDifference, mulDifference])
for ilayer, densedimension in enumerate(self.c['densedimension']):
allDifference = Dense(
units=int(densedimension),
activation=self.c['denseactivate'],
kernel_initializer=self.c['denseinitial'])(allDifference)
output = Dense(
name='output',
units=self.c['num_classes'],
activation='softmax',
kernel_initializer=self.c['denseinitial'])(allDifference)
self.model = Model(inputs=[input0, input1], outputs=output)
self.model.compile(loss='mean_squared_error',
optimizer=self.c['optimizer'])
def _rnncnn(x_shape, lr, conv_filters, kernel_size, pool_size, lstm_units,
dropout1, dropout2):
"""https://github.com/lykaust15/Deep_learning_examples/blob/master/2.CNN_
RNN_sequence_analysis/DNA_sequence_function_prediction.ipynb"""
model = Sequential([
Convolution1D(
activation='relu',
input_shape=(x_shape[1], x_shape[2]),
padding='valid',
filters=conv_filters,
kernel_size=kernel_size,
),
MaxPooling1D(pool_size=pool_size, strides=13),
Dropout(dropout1),
Bidirectional(LSTM(lstm_units, return_sequences=True), ),
Dropout(dropout2),
Flatten(),
Dense(input_dim=75 * 640, units=925),
Activation('relu'),
Dense(input_dim=925, units=1),
Activation('sigmoid'),
])
model.compile(loss='binary_crossentropy',
optimizer=keras.optimizers.Adam(lr),
metrics=['acc'])
return model
def create_model(sequence_length):
input_shape = (sequence_length, embedding_dim)
model_input = Input(shape=input_shape)
z = model_input
z = Dropout(dropout_prob[0])(z)
# Convolutional block
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
print(model.summary())
return model
def model(input_shape):
input_ = Input(input_shape)
# dilation_rate为1的一层因果卷积
X = Convolution1D(32, 2, padding='causal', dilation_rate=1)(input_)
# dilation_rate为2的一层扩张卷积
A, B = wavenetBlock(64, 2, 2)(X)
skip_connections = [B]
# 扩大倍数4,8,16,32,64,128,256,1,2,4,8,16,32,64,128,256,1,2,4,8的20层扩张卷积
for i in range(20):
A, B = wavenetBlock(64, 2, 2**((i + 2) % 9))(A)
skip_connections.append(B)
net = Add()(skip_connections)
net = Activation('relu')(net)
net = Convolution1D(128, 1, activation='relu')(net)
net = Convolution1D(256, 1)(net)
net = Activation('softmax')(net)
return Model(inputs=input_, outputs=net)
def f(input_):
residual = input_
h = Convolution1D(n_atrous_filters,
atrous_filter_size,
padding='same',
dilation_rate=atrous_rate)(input_)
tanh_out = Activation('tanh')(h)
s = Convolution1D(n_atrous_filters,
atrous_filter_size,
padding='same',
dilation_rate=atrous_rate)(input_)
sigmoid_out = Activation('sigmoid')(s)
merged = Multiply()([tanh_out, sigmoid_out])
skip_out = Convolution1D(1, 1, activation='relu',
padding='same')(merged)
out = Add()([skip_out, residual])
return out, skip_out
def __generate_base_model(self) -> Model:
sequence_input = Input(shape=(3000, 1))
# twice convolutional layer
sequence = Convolution1D(filters=32,
kernel_size=self.__kernel_size,
padding=self.__padding_valid,
activation=activations.relu)(sequence_input)
sequence = Convolution1D(filters=32,
kernel_size=self.__kernel_size,
padding=self.__padding_valid,
activation=activations.relu)(sequence)
for filters in [32, 32, 256]:
# max pool and dropout
sequence = MaxPool1D(pool_size=self.__pool_size,
padding=self.__padding_valid)(sequence)
sequence = SpatialDropout1D(rate=self.__dropout_rate)(sequence)
# twice convolutional layer again
sequence = Convolution1D(filters=filters,
kernel_size=self.__kernel_size,
padding=self.__padding_valid,
activation=activations.relu)(sequence)
sequence = Convolution1D(filters=filters,
kernel_size=self.__kernel_size,
padding=self.__padding_valid,
activation=activations.relu)(sequence)
# finale block
sequence = GlobalMaxPool1D()(sequence)
sequence = Dropout(rate=self.__dropout_rate)(sequence)
sequence = Dense(units=64, activation=activations.relu)(sequence)
# last dropout and model generation
model = models.Model(
inputs=sequence_input,
outputs=Dropout(rate=self.__dropout_rate)(sequence))
# compile model
model.compile(optimizer=optimizers.Adam(),
loss=losses.sparse_categorical_crossentropy,
metrics=self.__metrics)
return model
def cnn_model(
input_shape, # input list shape (word index list)
num_classes, # output shape
num_features, # number of word + empty
embedding_matrix,
filters,
kernel_sizes,
dropout_rate,
embedding_trainable,
l2_lambda):
embedding_layer = Embedding(
input_dim=num_features,
output_dim=300, # hard code
embeddings_initializer=Constant(embedding_matrix),
input_length=input_shape,
trainable=embedding_trainable)
# word index list, not map to embedding yet
sequence_input = Input(shape=(input_shape, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
nn_layers = list()
for kernel_size in kernel_sizes:
conv_layer_0 = Convolution1D(filters, kernel_size,
padding='valid')(embedded_sequences)
conv_layer_1 = BatchNormalization(axis=1)(conv_layer_0)
conv_layer_2 = Activation('relu')(conv_layer_1)
pool_layer_0 = MaxPooling1D(input_shape - kernel_size +
1)(conv_layer_2)
pool_layer_1 = Dropout(dropout_rate)(pool_layer_0)
nn_layers.append(pool_layer_1)
# merge diff kernal size generated output
line_merge_layer = concatenate(nn_layers)
line_flat_layer = Flatten()(line_merge_layer)
norm_layer = BatchNormalization(axis=1)(line_flat_layer)
drop_layer = Dropout(dropout_rate)(norm_layer)
preds = Dense(num_classes,
kernel_regularizer=regularizers.l2(l2_lambda),
activation='softmax')(drop_layer)
cnn_model = Model(inputs=sequence_input, outputs=preds)
return cnn_model
def create_model(sequence_length):
# Build model
input_shape = (sequence_length, )
model_input = Input(shape=input_shape)
z = Embedding(len(vocabulary_inv),
embedding_dim,
input_length=sequence_length,
name="embedding")(model_input)
z = Dropout(dropout_prob[0])(z)
# Convolutional block
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
# Initialize weights with word2vec
weights = np.array([v for v in embedding_weights.values()])
print("Initializing embedding layer with word2vec weights, shape",
weights.shape)
embedding_layer = model.get_layer("embedding")
embedding_layer.set_weights([weights])
print(model.summary())
return model
def __call__(self, inputs):
x = inputs
# 1D FCN.
x = Convolution1D(self.nb_filters, 1, padding=self.padding)(x)
skip_connections = []
for s in range(self.nb_stacks):
for d in self.dilations:
x, skip_out = residual_block(x,
dilation_rate=d,
nb_filters=self.nb_filters,
kernel_size=self.kernel_size,
padding=self.padding,
dropout_rate=self.dropout_rate)
skip_connections.append(skip_out)
if self.use_skip_connections:
x = keras.layers.add(skip_connections)
if not self.return_sequences:
x = Lambda(lambda tt: tt[:, -1, :])(x)
return x
def residual_block(x, s, i, activation, nb_filters, kernel_size):
original_x = x
conv = Conv1D(filters=nb_filters,
kernel_size=kernel_size,
dilation_rate=2**i,
padding='causal',
name='dilated_conv_%d_tanh_s%d' % (2**i, s))(x)
if activation == 'norm_relu':
x = Activation('relu')(conv)
x = Lambda(channel_normalization)(x)
elif activation == 'wavenet':
x = wave_net_activation(conv)
else:
x = Activation(activation)(conv)
x = SpatialDropout1D(0.05)(x)
# 1x1 conv.
x = Convolution1D(nb_filters, 1, padding='same')(x)
res_x = keras.layers.add([original_x, x])
return res_x, x
def train():
print("creating model")
X_1 = pickle.load(
open(datasetpath + "/mfcc_coefficients_training_vector.pickle", "rb"))
X_test_1 = pickle.load(
open(
datasetpath +
"/mfcc_coefficients_evaluation_training_vector.pickle", "rb"))
X_2 = pickle.load(
open(datasetpath + "/spectral-contrast_peaks_training_vector.pickle",
"rb"))
X_test_2 = pickle.load(
open(
datasetpath +
"/spectral-contrast_peaks_evaluation_training_vector.pickle",
"rb"))
print("X_1 (MFCC: items x max length x buckets)", X_1.shape)
print("X_test_1", X_test_1.shape)
print("X_2 (spectral contrast: items x peaks x buckets)", X_2.shape)
print("X_test_2", X_test_2.shape)
# crashes because input shape can not be generated using input_shapes = tf_utils.convert_shapes(inputs, to_tuples=False)
pool_length = 4
mfcc_shape = X_1.shape[1:] # conv on 2d
sc_shape = X_2.shape[1:] # conv on 2d
# create model
input_mfcc = keras.Input(shape=mfcc_shape, name="mfcc_input")
model_mfcc = Convolution1D(filters=100,
kernel_size=4,
padding='valid',
strides=1)(input_mfcc)
model_mfcc = Activation('relu')(model_mfcc)
model_mfcc = MaxPooling1D(pool_size=pool_length)(model_mfcc)
model_mfcc = Dropout(0.3)(model_mfcc)
model_mfcc = Convolution1D(filters=20,
kernel_size=4,
padding='valid',
strides=1)(model_mfcc)
model_mfcc = Activation('relu')(model_mfcc)
model_mfcc = MaxPooling1D(pool_size=pool_length)(model_mfcc)
model_mfcc = Dropout(0.3)(model_mfcc)
model_mfcc = GRU(300,
activation='sigmoid',
recurrent_activation='hard_sigmoid')(model_mfcc)
model_mfcc = Dropout(0.4)(model_mfcc)
inputs_scp = keras.Input(sc_shape, name="spectralconstrastpeaks")
model = Convolution1D(filters=100,
kernel_size=3,
padding='valid',
strides=4)(inputs_scp)
model = Activation('relu')(model)
model = MaxPooling1D(pool_size=1)(model)
model = Dropout(0.2)(model)
model_spc = GRU(
100,
activation='sigmoid',
recurrent_activation='hard_sigmoid',
# return_sequences=True
)(model)
model_spc = Dropout(0.2)(model_spc)
merged = keras.layers.Concatenate()([model_mfcc, model_spc])
x = keras.layers.BatchNormalization()(merged)
x = keras.layers.Dense(100, activation="elu")(x)
x = keras.layers.Dense(numGenres, activation='softmax')(x)
final_model = keras.Model(inputs=(input_mfcc, inputs_scp), outputs=x)
final_model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# write architecture to file
savedir = modelarchdir + "merged"
print(f"Saving architecture at {savedir}... ", end="")
final_model.save(savedir)
print("Ok")
tf.keras.utils.plot_model(final_model,
to_file='merged.png',
show_shapes=True)
print("Training")
if not os.path.exists("model_weights"):
os.makedirs("model_weights")
# check if can write to file because of the webserver might have reduced rights
weightpath = 'model_weights/mfcc_model_weights.hdf5'
f = open(weightpath, 'a')
f.write('test')
f.close()
os.remove(weightpath)
# #
# final_model.load_weights("model_weights/embeddings_10_sec_split_gztan_merged_model_weights.hdf5")
# #
# # for i in range(10):
# # print("epoch",i)
y = pickle.load(open(datasetpath + "/mfcc_coefficients_label.pickle",
"rb"))
y_test = pickle.load(
open(datasetpath + "/mfcc_coefficients_evaluation_label.pickle", "rb"))
print("y", y.shape)
print("y_test", y_test.shape)
if not os.path.exists("model_weights"):
os.makedirs("model_weights")
# checkpoint
filepath = "weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='max')
callbacks_list = [checkpoint]
x_test = (X_test_1, X_test_2)
history = final_model.fit(
(X_1, X_2),
y,
batch_size=batch_size,
epochs=nb_epoch,
validation_data=(x_test, y_test),
shuffle="batch",
callbacks=callbacks_list,
)
print("Saving final result.")
final_model.save_weights("model_weights/merged_model_weights.hdf5",
overwrite=True)
# dump training history
with open("experimental_results.json", "w") as f:
f.write(
json.dumps(history.history,
sort_keys=True,
indent=4,
separators=(',', ': ')))
for k, v in history.history.items():
_keys = list(history.history.keys())
_keys.sort()
plt.subplot(411 + _keys.index(k))
plt.title(k)
plt.plot(range(0, len(v)), v, marker="8", linewidth=1.5)
plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)
plt.savefig('history.png')
plot_confusion_matrix(final_model, y_test, x_test)
input_shape = (sequence_length, )
model_input = Input(shape=input_shape)
z = Embedding(len(vocabulary_inv),
embedding_dim,
input_length=sequence_length,
name="embedding")(model_input)
z = Dropout(dropout_prob[0])(z)
# Convolutional block
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(
conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy",
def dilated_tcn(num_feat,
num_classes,
nb_filters,
kernel_size,
dilations,
nb_stacks,
max_len,
activation='norm_relu',
use_skip_connections=True,
return_param_str=False,
output_slice_index=None,
regression=False,
lr=0.00007):
"""
dilation_depth : number of layers per stack
nb_stacks : number of stacks.
"""
input_layer = Input(name='input_layer', shape=(max_len, num_feat))
x = input_layer
x = Convolution1D(nb_filters,
kernel_size,
padding='causal',
name='initial_conv')(x)
skip_connections = []
for s in range(nb_stacks):
for i in dilations:
x, skip_out = residual_block(x, s, i, activation, nb_filters,
kernel_size)
skip_connections.append(skip_out)
if use_skip_connections:
x = keras.layers.add(skip_connections)
x = Activation('relu')(x)
if output_slice_index is not None: # can test with 0 or -1.
if output_slice_index == 'last':
output_slice_index = -1
if output_slice_index == 'first':
output_slice_index = 0
x = Lambda(lambda tt: tt[:, output_slice_index, :])(x)
print('x.shape=', x.shape)
if not regression:
# classification
x = Dense(num_classes)(x)
x = Activation('softmax', name='output_softmax')(x)
output_layer = x
print(f'model.x = {input_layer.shape}')
print(f'model.y = {output_layer.shape}')
model = Model(input_layer, output_layer)
adam = optimizers.Adam(lr=lr, clipnorm=1.)
model.compile(adam,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
print('Adam with norm clipping.')
else:
# regression
x = Dense(1)(x)
x = Activation('linear', name='output_dense')(x)
output_layer = x
print(f'model.x = {input_layer.shape}')
print(f'model.y = {output_layer.shape}')
model = Model(input_layer, output_layer)
adam = optimizers.Adam(lr=lr, clipnorm=1.)
model.compile(adam, loss='mean_squared_error', metrics=['mae', 'acc'])
if return_param_str:
param_str = 'D-TCN_C{}_B{}_L{}'.format(2, nb_stacks, dilations)
return model, param_str
else:
return model
def main(arg):
directory = Path('./saved_predictions/')
directory.mkdir(exist_ok=True)
directory = Path('./saved_models/')
directory.mkdir(exist_ok=True)
directory = Path('./training_checkpoints/')
directory.mkdir(exist_ok=True)
input_yx_size = tuple(args.input_yx_size)
batch_size = args.batch_size
epochs = args.epochs
learning_rate = args.learning_rate
num_test_samples = args.num_test_samples
save_weights = args.save_weights
every = args.every
num_samples = args.num_samples
save_train_prediction = args.save_train_prediction
save_test_prediction = args.save_test_prediction
verbose = args.verbose
validation_ratio = args.validation_ratio
y_axis_len, x_axis_len = input_yx_size
decay = args.decay
decay = args.decay
load_weights = args.load_weights
y_axis_len, x_axis_len = input_yx_size
num_points = y_axis_len * x_axis_len
is_flat_channel_in = args.is_flat_channel_in
input_points = Input(shape=(num_points, 4))
x = input_points
x = Convolution1D(64, 1, activation='relu', input_shape=(num_points, 4))(x)
x = BatchNormalization()(x)
x = Convolution1D(128, 1, activation='relu')(x)
x = BatchNormalization()(x)
x = Convolution1D(512, 1, activation='relu')(x)
x = BatchNormalization()(x)
x = MaxPooling1D(pool_size=num_points)(x)
x = Dense(512, activation='relu')(x)
x = BatchNormalization()(x)
x = Dense(256, activation='relu')(x)
x = BatchNormalization()(x)
x = Dense(16,
weights=[
np.zeros([256, 16]),
np.array([1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,
1]).astype(np.float32)
])(x)
input_T = Reshape((4, 4))(x)
# forward net
g = Lambda(mat_mul, arguments={'B': input_T})(input_points)
g = Convolution1D(64, 1, input_shape=(num_points, 3), activation='relu')(g)
g = BatchNormalization()(g)
g = Convolution1D(64, 1, input_shape=(num_points, 3), activation='relu')(g)
g = BatchNormalization()(g)
# feature transformation net
f = Convolution1D(64, 1, activation='relu')(g)
f = BatchNormalization()(f)
f = Convolution1D(128, 1, activation='relu')(f)
f = BatchNormalization()(f)
f = Convolution1D(128, 1, activation='relu')(f)
f = BatchNormalization()(f)
f = MaxPooling1D(pool_size=num_points)(f)
f = Dense(512, activation='relu')(f)
f = BatchNormalization()(f)
f = Dense(256, activation='relu')(f)
f = BatchNormalization()(f)
f = Dense(64 * 64,
weights=[
np.zeros([256, 64 * 64]),
np.eye(64).flatten().astype(np.float32)
])(f)
feature_T = Reshape((64, 64))(f)
# forward net
g = Lambda(mat_mul, arguments={'B': feature_T})(g)
seg_part1 = g
g = Convolution1D(64, 1, activation='relu')(g)
g = BatchNormalization()(g)
g = Convolution1D(32, 1, activation='relu')(g)
g = BatchNormalization()(g)
g = Convolution1D(32, 1, activation='relu')(g)
g = BatchNormalization()(g)
# global_feature
global_feature = MaxPooling1D(pool_size=num_points)(g)
global_feature = Lambda(exp_dim, arguments={'num_points':
num_points})(global_feature)
# point_net_seg
c = concatenate([seg_part1, global_feature])
""" c = Convolution1D(512, 1, activation='relu')(c)
c = BatchNormalization()(c)
c = Convolution1D(256, 1, activation='relu')(c)
c = BatchNormalization()(c)
c = Convolution1D(128, 1, activation='relu')(c)
c = BatchNormalization()(c)
c = Convolution1D(128, 1, activation='relu')(c)
c = BatchNormalization()(c) """
c = Convolution1D(256, 1, activation='relu')(c)
c = BatchNormalization()(c)
c = Convolution1D(128, 4, activation='relu', strides=4)(c)
c = BatchNormalization()(c)
c = Convolution1D(128, 4, activation='relu', strides=4)(c)
c = BatchNormalization()(c)
c = Convolution1D(128, 4, activation='relu', strides=4)(c)
c = BatchNormalization()(c)
c = Convolution1D(64, 4, activation='relu', strides=4)(c)
c = BatchNormalization()(c)
c = Convolution1D(64, 4, activation='relu', strides=4)(c)
c = BatchNormalization()(c)
c = Convolution1D(32, 1, activation='relu')(c)
c = BatchNormalization()(c)
""" c = Convolution1D(128, 4, activation='relu',strides=4)(c)
c = Convolution1D(64, 4, activation='relu',strides=4)(c)
c = Convolution1D(32, 4, activation='relu',strides=4)(c)
c = Convolution1D(16, 1, activation='relu')(c)
c = Convolution1D(1, 1, activation='relu')(c) """
#c = tf.keras.backend.squeeze(c,3);
c = CuDNNLSTM(64, return_sequences=False)(c)
#c =CuDNNLSTM(784, return_sequences=False))
#c =CuDNNLSTM(256, return_sequences=False))
#c = Reshape([16,16,1])(c)
c = Reshape([8, 8, 1])(c)
c = Conv2DTranspose(8, (3, 3),
padding="same",
activation="relu",
strides=(2, 2))(c)
c = Conv2DTranspose(8, (3, 3), padding="valid", activation="relu")(c)
#c =Dropout(0.4))
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(16, (3, 3), padding="valid", activation="relu")(c)
#c =Dropout(0.4))
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(32, (3, 3), padding="valid", activation="relu")(c)
#c =Dropout(0.4))
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(32, (3, 3), padding="valid", activation="relu")(c)
#c =Dropout(0.4))
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(32, (3, 3), padding="valid", activation="relu")(c)
#c =Dropout(0.4))
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(64, (3, 3), padding="valid", activation="relu")(c)
#c =Dropout(0.4))
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(64, (3, 3), padding="valid", activation="relu")(c)
#c =Dropout(0.4))
c = tf.keras.layers.BatchNormalization()(c)
#c =Dropout(0.4))
c = Conv2DTranspose(128, (3, 3),
padding="same",
activation="relu",
strides=(2, 2))(c)
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(128, (3, 3), padding="valid", activation="relu")(c)
#c =Dropout(0.4))
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(128, (3, 3),
padding="same",
activation="relu",
strides=(2, 2))(c)
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(128, (3, 3), padding="valid", activation="relu")(c)
c = tf.keras.layers.BatchNormalization()(c)
#c =Dropout(0.4))
#c =tf.keras.layers.BatchNormalization())
c = Conv2DTranspose(64, (3, 3), padding="same", strides=(4, 2))(c)
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(32, (3, 3), padding="valid", activation="relu")(c)
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(32, (3, 3), padding="valid", activation="relu")(c)
c = tf.keras.layers.BatchNormalization()(c)
#c =Dropout(0.4))
c = Conv2DTranspose(32, (3, 3),
padding="same",
activation="relu",
strides=(1, 1))(c)
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(32, (3, 1), padding="valid", activation="relu")(c)
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(32, (3, 1), padding="valid", activation="relu")(c)
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(16, (1, 1), padding="valid", activation="relu")(c)
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(8, (1, 1), padding="valid", activation="relu")(c)
c = tf.keras.layers.BatchNormalization()(c)
c = Conv2DTranspose(1, (1, 1), padding="valid")(c)
""" c =Conv2DTranspose(4, (1,1),padding="same",activation="relu"))
c =Conv2DTranspose(2, (1,1),padding="same",activation="relu"))
#c =Dropout(0.4))
c =Conv2DTranspose(1, (1,1),padding="same")) """
prediction = tf.keras.layers.Reshape([512, 256])(c)
""" c1 ,c2 = tf.split(c,[256,256],axis=1,name="split")
complexNum = tf.dtypes.complex(
c1,
c2,
name=None
)
complexNum =tf.signal.ifft2d(
complexNum,
name="IFFT"
)
real = tf.math.real(complexNum)
imag = tf.math.imag(complexNum)
con = concatenate([real,imag])
prediction =tf.keras.layers.Reshape([ 512, 256])(con)
"""
# define model
model = Model(inputs=input_points, outputs=prediction)
opt = tf.keras.optimizers.Adam(lr=learning_rate, decay=decay)
loss = tf.keras.losses.MeanSquaredError()
mertric = ['mse']
if args.loss is "MAE":
loss = tf.keras.losses.MeanAbsoluteError()
mertric = ['mae']
model.compile(
loss=loss,
optimizer=opt,
metrics=mertric,
)
model.summary()
if load_weights:
model.load_weights('./training_checkpoints/cp-best_loss.ckpt')
#edit data_loader.py if you want to play with data
input_ks, ground_truth = load_data(num_samples,
is_flat_channel_in=is_flat_channel_in)
input_ks = input_ks / np.max(input_ks)
checkpoint_path = "./training_checkpoints/cp-{epoch:04d}.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
# Create checkpoint callback
#do you want to save the model's wieghts? if so set this varaible to true
cp_callback = []
NAME = "NUFFT_NET"
tensorboard = TensorBoard(log_dir="logs/{}".format(NAME))
cp_callback.append(tensorboard)
if save_weights:
cp_callback.append(
tf.keras.callbacks.ModelCheckpoint(checkpoint_dir,
save_weights_only=True,
verbose=verbose,
period=every))
if args.is_train:
model.fit(input_ks,
ground_truth,
batch_size=batch_size,
epochs=epochs,
validation_split=validation_ratio,
callbacks=cp_callback)
if args.name_model is not "":
model.save('./saved_mdoels/' + args.name_model)
dict_name = './saved_predictions/'
#return to image size
x_axis_len = int(x_axis_len / 4)
np.random.seed(int(time()))
if save_train_prediction <= num_samples:
rand_ix = np.random.randint(0, num_samples - 1, save_train_prediction)
#kspace = np.zeros((save_train_prediction,
#y_axis_len,input_ks[rand_ix].shape[1]))
kspace = input_ks[rand_ix]
if args.save_input:
np.save("./saved_predictions/inputs.npy", input_ks[rand_ix])
ground_truth = ground_truth[rand_ix]
preds = model.predict(kspace, batch_size=save_train_prediction)
for i in range(save_train_prediction):
output = np.reshape(preds[i], (y_axis_len * 2, x_axis_len))
output = output * 255
output[np.newaxis, ...]
output_gt = ground_truth[i]
output_gt[np.newaxis, ...]
output = np.concatenate([output, output_gt], axis=0)
np.save(dict_name + 'prediction%d.npy' % (i + 1), output)
input_ks, ground_truth = load_data(
num_test_samples, 'test', is_flat_channel_in=is_flat_channel_in)
input_ks = input_ks / np.max(input_ks)
if args.is_eval:
model.evaluate(input_ks,
ground_truth,
batch_size,
verbose,
callbacks=cp_callback)
if save_test_prediction <= num_test_samples:
rand_ix = np.random.randint(0, num_test_samples - 1,
save_test_prediction)
kspace = input_ks[rand_ix]
if args.save_input:
np.save("./saved_predictions/test_inputs.npy", input_ks[rand_ix])
ground_truth = ground_truth[rand_ix]
preds = model.predict(kspace, batch_size=save_test_prediction)
for i in range(save_test_prediction):
output = np.reshape(preds[i], (y_axis_len * 2, x_axis_len))
output = output * 255
output[np.newaxis, ...]
output_gt = ground_truth[i]
output_gt[np.newaxis, ...]
output = np.concatenate([output, output_gt], axis=0)
np.save(dict_name + 'test_prediction%d.npy' % (i + 1), output)