def create_model(self, hyper_parameters):
"""
构建神经网络
:param hyper_parameters:json, hyper parameters of network
:return: tensor, moedl
"""
super().create_model(hyper_parameters)
x = self.word_embedding.output
# x = Reshape((self.len_max, self.embed_size, 1))(embedding_output) # (None, 50, 30, 1)
# cnn + pool
for char_cnn_size in self.char_cnn_layers:
x = Convolution1D(filters = char_cnn_size[0],
kernel_size = char_cnn_size[1],)(x)
x = ThresholdedReLU(self.threshold)(x)
if char_cnn_size[2] != -1:
x = MaxPooling1D(pool_size = char_cnn_size[2],
strides = 1)(x)
x = Flatten()(x)
# full-connect
for full in self.full_connect_layers:
x = Dense(units=full,)(x)
x = ThresholdedReLU(self.threshold)(x)
x = Dropout(self.dropout)(x)
output = Dense(units=self.label, activation=self.activate_classify)(x)
self.model = Model(inputs=self.word_embedding.input, outputs=output)
self.model.summary(120)
def _build_model(self):
print "Building the model..."
# building the model
# Input layer
inputs = Input(shape=(self.l0, ), name='sent_input', dtype='int64')
# Embedding layer
x = Embedding(self.alphabet_size + 1,
self.embedding_size,
input_length=self.l0)(inputs)
# Convolution layers
for cl in self.conv_layers:
x = Convolution1D(cl[0], cl[1])(x)
x = ThresholdedReLU(self.th)(x)
if not cl[2] is None:
x = MaxPooling1D(cl[2])(x)
x = Flatten()(x)
# Fully connected layers
for idx, fl in enumerate(self.fully_layers):
x = Dense(fl, name="output" + str(idx))(x)
x = ThresholdedReLU(self.th)(x)
x = Dropout(self.dropout_p)(x)
predictions = Dense(self.no_of_classes, activation='softmax')(x)
self.model = Model(input=inputs, output=predictions)
optimizer = Adam()
self.model.compile(optimizer=optimizer,
loss='categorical_crossentropy')
print "Built"
def define_model_2(conv_layers,
fully_connected_layers,
threshold=1e-6,
num_classes=2,
optimizer='adam',
dropout_proba=0.5,
alphabet_size=ALPHABET_SIZE,
embedding_size=32,
input_size=MAX_INPUT_LEN,
loss='categorical_crossentropy'):
"""
Based on https://arxiv.org/abs/1509.01626
"""
inputs = Input(shape=(input_size, ), name='input_layer', dtype='int64')
x = Embedding(alphabet_size + 1, embedding_size,
input_length=input_size)(inputs)
for num_filters, filter_width, max_pool in conv_layers:
x = Convolution1D(filters=num_filters, kernel_size=filter_width)(x)
x = ThresholdedReLU(threshold)(x)
if max_pool != -1:
x = MaxPooling1D(max_pool)(x)
x = Flatten()(x)
for units in fully_connected_layers:
x = Dense(units)(x)
x = ThresholdedReLU(threshold)(x)
x = Dropout(dropout_proba)(x)
predictions = Dense(num_classes, activation='softmax')(x)
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])
return model
def create_model(input_size, alphabet_size, conv_layers, fc_layers,
num_of_classes):
# Input layer
inputs = Input(shape=(input_size, ), name='sent_input', dtype='int64')
# Embedding layers
x = Embedding(len(alphabet) + 1, 128, input_length=input_size)(inputs)
# 1D Convolutional layers
for cl in conv_layers:
x = Convolution1D(cl[0], cl[1])(x)
x = ThresholdedReLU(1e-6)(x)
if cl[2] != -1:
x = MaxPooling1D(cl[2])(x)
x = Flatten()(x)
# Fully Connected layers
for fl in fc_layers:
x = Dense(fl)(x)
x = ThresholdedReLU(1e-6)(x)
x = Dropout(0.5)(x)
# Output layer
predictions = Dense(num_of_classes, activation='softmax')(x)
return inputs, predictions
def _build_model(self):
# Input layer
inputs = Input(shape=(self.input_size, ), dtype='int64')
# Embedding layers
x = Embedding(self.alphabet_size + 1,
self.embedding_size,
input_length=self.input_size)(inputs)
# Convolution layers
for cl in self.conv_layers:
x = Convolution1D(cl[0], cl[1])(x)
x = ThresholdedReLU(self.threshold)(x)
if cl[2] != -1:
x = MaxPooling1D(cl[2])(x)
x = Flatten()(x)
# Fully connected layers
for fl in self.fully_connected_layers:
x = Dense(fl)(x)
x = ThresholdedReLU(self.threshold)(x)
x = Dropout(self.dropout_p)(x)
# Output layer
predictions = Dense(self.n_classes, activation='softmax')(x)
# Build and compile model
print("Compiling model...")
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer=self.optimizer,
loss=self.loss,
metrics=['accuracy'])
self.model = model
self.model.summary()
def _build_model(self):
"""
Build and compile the Character Level CNN model
Returns: None
"""
# Input layer
inputs = Input(shape=(self.input_size, ),
name='sent_input',
dtype='int64')
# Embedding layers
x = Embedding(self.alphabet_size + 1,
self.embedding_size,
input_length=self.input_size)(inputs)
# Convolution layers
for cl in self.conv_layers:
x = Convolution1D(cl[0], cl[1])(x)
x = ThresholdedReLU(self.threshold)(x)
if cl[2] != -1:
x = MaxPooling1D(cl[2])(x)
x = Flatten()(x)
# Fully connected layers
for fl in self.fully_connected_layers:
x = Dense(fl)(x)
x = ThresholdedReLU(self.threshold)(x)
x = Dropout(self.dropout_p)(x)
# Output layer
predictions = Dense(1, activation='sigmoid')(x)
# Build and compile model
model = Model(inputs=inputs, outputs=predictions)
self.model = model
return self.model
def average_dice_coef(y_true, y_pred):
y_pred = ThresholdedReLU(0.5)(y_pred)
loss = 0
label_length = y_pred.get_shape().as_list()[-1]
for num_label in range(label_length):
y_true_f = K.flatten(y_true[..., num_label])
y_pred_f = K.flatten(y_pred[..., num_label])
intersection = K.sum(y_true_f * y_pred_f)
loss += (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
return loss / label_length # 1>= loss >0
def KQV15(columns):
model = Sequential()
model.add(
Dense(int(columns / 1.5), input_dim=columns, activation='relu'))
model.add(ThresholdedReLU())
model.add(Dense(int(columns / 2), activation='relu'))
model.add(ThresholdedReLU())
model.add(Dense(int(columns / 5), activation='relu'))
model.add(Dense(1))
model.compile(loss="mean_squared_error",
optimizer=keras.optimizers.Adadelta(),
metrics=[tef.max_error, tef.mean_diff])
return model
def _build_model(self):
"""
Build and compile the Character Level CNN model
Returns: None
"""
# Input first layer
char_inputs = Input(shape=(self.input_size, ),
name='sent_char_input',
dtype='int64')
# Input second layer
word_inputs = Input(shape=(500, 200),
name='sent_word_input',
dtype='float')
# Embedding first layers
y = Embedding(self.alphabet_size + 1,
self.embedding_size,
input_length=self.input_size)(char_inputs)
# Concat two layers
x = Concatenate(axis=1)([y, word_inputs])
# Convolution layers
for cl in self.conv_layers:
x = Convolution1D(cl[0], cl[1])(x)
x = ThresholdedReLU(self.threshold)(x)
if cl[2] != -1:
x = MaxPooling1D(cl[2])(x)
x = LSTM(256,
dropout_W=0.2,
dropout_U=0.2,
input_shape=(3500, 200),
return_sequences=True)(x)
x = Flatten()(x)
# Fully connected layers
for fl in self.fully_connected_layers:
x = Dense(fl)(x)
x = ThresholdedReLU(self.threshold)(x)
x = Dropout(self.dropout_p)(x)
# Output layer
predictions = Dense(self.num_of_classes, activation='softmax')(x)
# Build and compile model
model = Model(inputs=[char_inputs, word_inputs], outputs=predictions)
model.compile(optimizer=self.optimizer,
loss=self.loss,
metrics=['accuracy'])
self.model = model
print("CharCNNZhang model built: ")
self.model.summary()
def create_final_block() -> None:
global prev_layer
activation = get_activation(config['final_activation'])
l_name = "CONV_FINAL"
conv = Conv3D(filters=1,
kernel_size=config['final_kernel_size'],
activation=activation,
padding=config['padding_mode'],
kernel_initializer=config['final_kernel_initializer'],
kernel_regularizer=None,
name=l_name)(prev_layer)
record_layer(l_name, conv)
if activation == 'linear':
l_name = l_name + "_A"
conv = config['final_activation'](name=l_name)(prev_layer)
record_layer(l_name, conv)
if config['labels'] > 1:
# TODO: Allow for multiple output labels
pass
if config['theta_cutoff'] is not None:
l_name = "OUT"
out = ThresholdedReLU(theta=config['theta_cutoff'],
name=l_name)(prev_layer)
record_layer(l_name, out)
def generate_embeddings_hard_attetion(config, incoming_layer):
if config['bidirectional_attention']:
gru = Bidirectional(
GRU(config['gru_units'],
return_sequences=True,
dropout=config['gru_dropout'],
recurrent_dropout=config['recurrent_dropout']))(incoming_layer)
else:
gru = GRU(
config['gru_units'],
return_sequences=True,
dropout=config['gru_dropout'],
recurrent_dropout=config['recurrent_dropout'])(incoming_layer)
dense_att_1 = TimeDistributed(
Dense(config['gru_units'], name='dense_att_1'))(gru)
att_1_act = ThresholdedReLU(theta=0.8)(dense_att_1)
# total units = 1 * INPUT_ACTIONS
dense_att_2 = TimeDistributed(Dense(1))(att_1_act)
# to undo the time distribution and have 1 value for each action
reshape_distributed = Reshape((config['max_phrase_length'], ))(dense_att_2)
attention = Activation('softmax')(reshape_distributed)
# so we can multiply it with embeddings
reshape_att = Reshape((config['max_phrase_length'], 1),
name='reshape_att')(attention)
# apply the attention to the embeddings
apply_att = Multiply()([incoming_layer, reshape_att])
return apply_att
def build_model():
model = Sequential()
model.add(Embedding(128, 128, input_length = 75))
model.add(Conv1D(filters=128, kernel_size=3, padding='same', strides=1))
model.add(ThresholdedReLU(1e-6))
model.add(MaxPooling1D(pool_size=2, padding='same'))
model.add(Conv1D(filters=128, kernel_size=2, padding='same', strides=1))
model.add(ThresholdedReLU(1e-6))
model.add(MaxPooling1D(pool_size=2, padding='same'))
model.add(Flatten())
model.add(Dense(64))
model.add(ThresholdedReLU(1e-6))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics = ['accuracy'])
#model.summary()
return model
def avg_dice_2(y_true, y_pred):
y_pred = ThresholdedReLU(0.5)(y_pred)
loss = 0
num_label = 2
y_true_f = K.flatten(y_true[..., num_label])
y_pred_f = K.flatten(y_pred[..., num_label])
intersection = K.sum(y_true_f * y_pred_f)
loss += (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
return loss # 1>= loss >0
def DefaultActivation():
if activation == "leaky-relu":
return LeakyReLU()
elif activation == "prelu":
return PReLU()
elif activation == "threshold-relu":
return ThresholdedReLU()
else:
return Activation(activation)
def train():
# load the kinetics classes
# datapath='/users/kevin/downloads/aicure-dataset/*/*.npy'
datapath = r'C:\Users\Chris\Documents\projects\cs172b\aicure-dataset\*\*.npy'
indexes, data, labels = load_data(datapath)
base_model = Inception_Inflated3d(weights='rgb_imagenet_and_kinetics',
include_top=False,
input_shape=(NUM_FRAMES, FRAME_HEIGHT,
FRAME_WIDTH,
NUM_RGB_CHANNELS))
output = Dropout(0.5)(base_model.output)
predict = Reshape((-1, 1024))(output)
#predict = AveragePooling1D(pool_size=3)(predict)
predict = Dense(NUM_FRAMES,
kernel_initializer='normal',
activation='sigmoid')(predict)
predict = ThresholdedReLU(theta=0.8, trainable=False)(predict)
model = Model(inputs=base_model.input, outputs=predict)
# freeze the first 100 layers
for layer in model.layers[:100]:
layer.trainable = False
# randomize the weights for the remaining trainable layers
# for layer in model.layers[150:195]: # change to 150:195 later
# layer.kernel_initializer = 'glorot_uniform'
optimizer = keras.optimizers.Adam(lr=1e-4)
model.compile(optimizer=optimizer, loss='mae', metrics=['accuracy'])
#model.summary()
train_indexes = indexes[:int(0.7 * len(indexes))]
validation_indexes = indexes[int(0.7 * len(indexes)):]
reducelr = tf.keras.callbacks.ReduceLROnPlateau(monitor='accuracy',
patience=2,
factor=0.2,
min_lr=1e-8)
earlystop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy',
patience=5)
callbacks = [reducelr, earlystop]
history = model.fit_generator(data_generator(data, labels, train_indexes,
BATCH_SIZE),
steps_per_epoch=int(STEPS * .7),
epochs=50,
validation_data=data_generator(
data, labels, validation_indexes),
validation_steps=int(STEPS * .3),
callbacks=callbacks)
write_out(history, 'hist.csv')
save_model(model, 'i3d')
def simple_block_2d(input,
number_of_filters,
downsample=False,
upsample=False,
convolution_kernel_size=(3, 3),
deconvolution_kernel_size=(2, 2),
weight_decay=0.0,
dropout_rate=0.0):
number_of_output_filters = number_of_filters
output = BatchNormalization()(input)
output = ThresholdedReLU(theta=0)(output)
if downsample:
output = MaxPooling2D(pool_size=(2, 2))(output)
output = Conv2D(
filters=number_of_filters,
kernel_size=convolution_kernel_size,
padding='same',
kernel_regularizer=regularizers.l2(weight_decay))(output)
if upsample:
output = Conv2DTranspose(
filters=number_of_filters,
kernel_size=deconvolution_kernel_size,
padding='same',
kernel_initializer=initializers.he_normal(),
kernel_regularizer=regularizers.l2(weight_decay))(output)
output = UpSampling2D(size=(2, 2))(output)
if dropout_rate > 0.0:
output = Dropout(rate=dropout_rate)(output)
# Modify the input so that it has the same size as the output
if downsample:
input = Conv2D(filters=number_of_output_filters,
kernel_size=(1, 1),
strides=(2, 2),
padding='same')(input)
elif upsample:
input = Conv2DTranspose(filters=number_of_output_filters,
kernel_size=(1, 1),
padding='same')(input)
input = UpSampling2D(size=(2, 2))(input)
elif number_of_filters != number_of_output_filters:
input = Conv2D(filters=number_of_output_filters,
kernel_size=(1, 1),
padding='same')(input)
output = skip_connection(input, output)
return (output)
def get_activation_layer(activation):
if activation == 'LeakyReLU':
return LeakyReLU()
if activation == 'PReLU':
return PReLU()
if activation == 'ELU':
return ELU()
if activation == 'ThresholdedReLU':
return ThresholdedReLU()
return Activation(activation)
def create_model2(): m = Sequential() # (Conv -> Relu -> Conv -> Relu -> MaxPool) * 3 -> Flat -> Dense for u in range(3): if u == 0: m.add(Conv2D(32, (3, 3), activation='relu', input_shape=(150, 100, 3))) else: m.add(Conv2D(32, (3, 3), activation='relu')) m.add(ThresholdedReLU(0)) m.add(Conv2D(32, (3, 3), activation='relu')) m.add(ThresholdedReLU(0)) m.add(MaxPooling2D(pool_size=(2, 2))) m.add(Flatten()) m.add(Dense(num_train_classes, activation='softmax')) return m
def _build_model(self):
"""
Build and compile the Character Level CNN model
Returns: None
"""
# Input layer
inputs = Input(shape=(self.input_size, ),
name='sent_input',
dtype='int64')
# Embedding layers
x = Embedding(self.alphabet_size + 1,
self.embedding_size,
input_length=self.input_size)(inputs)
# Convolution layers
for cl in self.conv_layers:
x = Convolution1D(cl[0], cl[1])(x)
x = ThresholdedReLU(self.threshold)(x)
if cl[2] != -1:
x = MaxPooling1D(cl[2])(x)
x = Flatten()(x)
# Fully connected layers
for fl in self.fully_connected_layers:
x = Dense(fl)(x)
x = ThresholdedReLU(self.threshold)(x)
x = Dropout(self.dropout_p)(x)
# Output layer
predictions = Dense(self.num_of_classes, activation='softmax')(x)
# Build and compile model
model = Model(inputs=inputs, outputs=predictions)
# model.compile(optimizer=self.optimizer, loss=self.loss, metrics=[f1_m, recall_m, precision_m])
model.compile(optimizer=RAdam(learning_rate=self.LR),
loss=self.loss,
metrics=[f1_m, recall_m, precision_m])
self.model = model
print("CharCNNZhang model built: ")
self.model.summary()
def create_model_vgg16(): from keras.applications.vgg16 import VGG16 input_tensor = Input(shape=(150, 100, 3)) base_model = VGG16(weights='imagenet', include_top=False, input_tensor=input_tensor) for layer in base_model.layers[:15]: layer.trainable = False top_model = Sequential() top_model.add(ZeroPadding2D((1, 1), input_shape=base_model.output_shape[1:])) top_model.add(Conv2D(32, (3, 3), activation='relu')) top_model.add(BatchNormalization()) top_model.add(ThresholdedReLU(0)) top_model.add(ZeroPadding2D((1, 1))) top_model.add(Conv2D(32, (3, 3), activation='relu')) top_model.add(BatchNormalization()) top_model.add(ThresholdedReLU(0)) top_model.add(ZeroPadding2D((1, 1))) top_model.add(Conv2D(32, (3, 3), activation='relu')) top_model.add(BatchNormalization()) top_model.add(ThresholdedReLU(0)) top_model.add(MaxPooling2D(pool_size=(2, 2))) top_model.add(Flatten()) top_model.add(BatchNormalization()) top_model.add(Dense(num_train_classes, activation='softmax')) m = Model(inputs=base_model.input, outputs=top_model(base_model.output)) # m.summary() return m
def __init__(self, inputLength, embedding_weights):
print("init................................................")
input_document_a = Input(shape=(inputLength, ),
name="input_document_a")
input_document_b = Input(shape=(inputLength, ),
name="input_document_b")
embedding_a = Embedding(input_dim=embedding_weights.shape[0],
output_dim=embedding_weights.shape[1],
input_length=inputLength,
name="embedding_a",
weights=[embedding_weights],
trainable=False)(input_document_a)
embedding_b = Embedding(input_dim=embedding_weights.shape[0],
output_dim=embedding_weights.shape[1],
input_length=inputLength,
name="embedding_b",
weights=[embedding_weights],
trainable=False)(input_document_b)
lstm_a = LSTM(units=embedding_weights.shape[1],
dropout=0.2,
name="lstm_a")(embedding_a)
lstm_b = LSTM(units=embedding_weights.shape[1],
dropout=0.2,
name="lstm_b")(embedding_b)
concate = Concatenate(input_shape=(embedding_weights.shape[1], ),
name="concate")([lstm_a, lstm_b])
dense = Dense(units=inputLength,
input_shape=(2 * inputLength, ),
name="dense1")(concate)
activation = Activation(input_shape=(inputLength, ),
activation="sigmoid",
name="activation")(dense)
dense2 = Dense(units=1,
input_shape=(inputLength, ),
activation="softmax",
name="dense2")(activation)
# output = ThresholdedBinary(name="output")(dense2)
output = ThresholdedReLU(theta=0.5, name="output")(dense2)
self.model = Model(inputs=[input_document_a, input_document_b],
outputs=output)
def create_model(input_size):
model = Sequential()
model.add(
LSTM(200,
batch_input_shape=(None, 1, input_size),
activation='softsign',
recurrent_activation='hard_sigmoid',
return_sequences=True))
model.add(
LSTM(100,
batch_input_shape=(None, 1, input_size),
activation='softsign',
recurrent_activation='hard_sigmoid',
return_sequences=True))
model.add(
LSTM(80,
batch_input_shape=(None, 1, input_size),
activation='softsign',
recurrent_activation='hard_sigmoid',
return_sequences=True))
model.add(
LSTM(60,
batch_input_shape=(None, 1, input_size),
activation='softsign',
recurrent_activation='hard_sigmoid',
return_sequences=False))
model.add(Dense(60))
model.add(Activation('softplus'))
model.add(Dense(40))
model.add(ThresholdedReLU())
model.add(Dense(20))
model.add(Activation('sigmoid'))
model.add(Dense(2))
model.add(Activation('hard_sigmoid'))
model.compile(loss='cosine_proximity',
optimizer=RMSprop(),
metrics=['acc'])
return model
def get(activation, nb_features=0, **kwargs):
if (activation == 'WiG_Dense'):
if (not 'kernel_initializer' in kwargs):
kwargs['kernel_initializer'] = 'zeros'
return WiG_Dense(nb_features, **kwargs)
elif (activation == 'WiG_Conv2D'):
if (not 'kernel_size' in kwargs):
kwargs['kernel_size'] = (3, 3)
if (not 'padding' in kwargs):
kwargs['padding'] = 'same'
if (not 'kernel_initializer' in kwargs):
kwargs['kernel_initializer'] = 'zeros'
return WiG_Conv2D(nb_features, **kwargs)
elif (activation == 'LeakyReLU'):
return LeakyReLU(**kwargs)
elif (activation == 'PReLU'):
return PReLU(**kwargs)
elif (activation == 'ELU'):
return ELU(**kwargs)
elif (activation == 'ThresholdedReLU'):
return ThresholdedReLU(**kwargs)
elif (activation == 'Swish'):
return Swish(**kwargs)
elif (activation == 'SiL'):
return SiL(**kwargs)
return Activation(activation, **kwargs)
print("Loaded")
print("Building the model..."),
# building the model
# Input layer
inputs = Input(shape=(l0, ), name='sent_input', dtype='int64')
# Embedding layer
x = Embedding(alphabet_size + 1, embedding_size, input_length=l0)(inputs)
# Convolution layers
for cl in conv_layers:
x = Convolution1D(cl[0], cl[1])(x)
x = ThresholdedReLU(th)(x)
if not cl[2] is None:
x = MaxPooling1D(cl[2])(x)
x = Flatten()(x)
#Fully connected layers
for fl in fully_layers:
x = Dense(fl)(x)
x = ThresholdedReLU(th)(x)
x = Dropout(0.5)(x)
predictions = Dense(num_of_classes, activation='softmax')(x)
model = Model(input=inputs, output=predictions)
def test_delete_channels_advanced_activations(channel_index, data_format):
layer_test_helper_flatten_2d(LeakyReLU(), channel_index, data_format)
layer_test_helper_flatten_2d(ELU(), channel_index, data_format)
layer_test_helper_flatten_2d(ThresholdedReLU(), channel_index, data_format)
def create_resunet_model_3d(input_image_size,
number_of_outputs=1,
number_of_filters_at_base_layer=32,
bottle_neck_block_depth_schedule=(3, 4),
convolution_kernel_size=(3, 3, 3),
deconvolution_kernel_size=(2, 2, 2),
dropout_rate=0.0,
weight_decay=0.0,
mode='classification'):
"""
3-D implementation of the Resnet + U-net deep learning architecture.
Creates a keras model of the U-net + ResNet deep learning architecture for
image segmentation and regression with the paper available here:
https://arxiv.org/abs/1608.04117
This particular implementation was ported from the following python
implementation:
https://github.com/veugene/fcn_maker/
Arguments
---------
input_image_size : tuple of length 4
Used for specifying the input tensor shape. The
shape (or dimension) of that tensor is the image dimensions followed by
the number of channels (e.g., red, green, and blue). The batch size
(i.e., number of training images) is not specified a priori.
number_of_outputs : integer
Meaning depends on the mode. For 'classification' this is the number of
segmentation labels. For 'regression' this is the number of outputs.
number_of_filters_at_base_layer : integer
Number of filters at the beginning and end of the 'U'. Doubles at each
descending/ascending layer.
bottle_neck_block_depth_schedule : tuple
Tuple that provides the encoding layer schedule for the number of bottleneck
blocks per long skip connection.
convolution_kernel_size : tuple of length 3
3-d vector defining the kernel size during the encoding path
deconvolution_kernel_size : tuple of length 3
3-d vector defining the kernel size during the decoding
dropout_rate : scalar
Float between 0 and 1 to use between dense layers.
weight_decay : scalar
Weighting parameter for L2 regularization of the kernel weights of the
convolution layers. Default = 0.0.
mode : string
'classification' or 'regression'. Default = 'classification'.
Returns
-------
Keras model
A 3-D Keras model defining the network.
Example
-------
>>> model = create_resunet_model_3d((128, 128, 128, 1))
>>> model.summary()
"""
def simple_block_3d(input,
number_of_filters,
downsample=False,
upsample=False,
convolution_kernel_size=(3, 3, 3),
deconvolution_kernel_size=(2, 2, 2),
weight_decay=0.0,
dropout_rate=0.0):
number_of_output_filters = number_of_filters
output = BatchNormalization()(input)
output = ThresholdedReLU(theta=0)(output)
if downsample:
output = MaxPooling3D(pool_size=(2, 2, 2))(output)
output = Conv3D(
filters=number_of_filters,
kernel_size=convolution_kernel_size,
padding='same',
kernel_regularizer=regularizers.l2(weight_decay))(output)
if upsample:
output = Conv3DTranspose(
filters=number_of_filters,
kernel_size=deconvolution_kernel_size,
padding='same',
kernel_initializer=initializers.he_normal(),
kernel_regularizer=regularizers.l2(weight_decay))(output)
output = UpSampling3D(size=(2, 2, 2))(output)
if dropout_rate > 0.0:
output = Dropout(rate=dropout_rate)(output)
# Modify the input so that it has the same size as the output
if downsample:
input = Conv3D(filters=number_of_output_filters,
kernel_size=(1, 1, 1),
strides=(2, 2, 2),
padding='same')(input)
elif upsample:
input = Conv3DTranspose(filters=number_of_output_filters,
kernel_size=(1, 1, 1),
padding='same')(input)
input = UpSampling3D(size=(2, 2, 2))(input)
elif number_of_filters != number_of_output_filters:
input = Conv3D(filters=number_of_output_filters,
kernel_size=(1, 1, 1),
padding='same')(input)
output = skip_connection(input, output)
return (output)
def bottle_neck_block_3d(input,
number_of_filters,
downsample=False,
upsample=False,
deconvolution_kernel_size=(2, 2, 2),
weight_decay=0.0,
dropout_rate=0.0):
output = input
number_of_output_filters = number_of_filters
if downsample:
output = BatchNormalization()(output)
output = ThresholdedReLU(theta=0)(output)
output = Conv3D(
filters=number_of_filters,
kernel_size=(1, 1, 1),
strides=(2, 2, 2),
kernel_initializer=initializers.he_normal(),
kernel_regularizer=regularizers.l2(weight_decay))(output)
output = BatchNormalization()(output)
output = ThresholdedReLU(theta=0)(output)
output = Conv3D(
filters=number_of_filters,
kernel_size=(1, 1, 1),
kernel_initializer=initializers.he_normal(),
kernel_regularizer=regularizers.l2(weight_decay))(output)
output = BatchNormalization()(output)
output = ThresholdedReLU(theta=0)(output)
if upsample:
output = Conv3DTranspose(
filters=number_of_filters,
kernel_size=deconvolution_kernel_size,
padding='same',
kernel_initializer=initializers.he_normal(),
kernel_regularizer=regularizers.l2(weight_decay))(output)
output = UpSampling3D(size=(2, 2, 2))(output)
output = Conv3D(
filters=(number_of_filters * 4),
kernel_size=(1, 1, 1),
kernel_initializer=initializers.he_normal(),
kernel_regularizer=regularizers.l2(weight_decay))(output)
number_of_output_filters = number_of_filters * 4
if dropout_rate > 0.0:
output = Dropout(rate=dropout_rate)(output)
# Modify the input so that it has the same size as the output
if downsample:
input = Conv3D(filters=number_of_output_filters,
kernel_size=(1, 1, 1),
strides=(2, 2, 2),
padding='same')(input)
elif upsample:
input = Conv3DTranspose(filters=number_of_output_filters,
kernel_size=(1, 1, 1),
padding='same')(input)
input = UpSampling3D(size=(2, 2, 2))(input)
elif number_of_filters != number_of_output_filters:
input = Conv3D(filters=number_of_output_filters,
kernel_size=(1, 1, 1),
padding='valid')(input)
output = skip_connection(input, output)
return (output)
def skip_connection(source, target, merge_mode='sum'):
layer_list = [source, target]
output = None
if merge_mode == 'sum':
output = Add()(layer_list)
else:
channel_axis = 0
if K.image_data_format() == 'channels_last':
channel_axis = -1
output = Concatenate(axis=channel_axis)(layer_list)
return (output)
inputs = Input(shape=input_image_size)
encoding_layers_with_long_skip_connections = []
encoding_layer_count = 1
# Preprocessing layer
model = Conv3D(filters=number_of_filters_at_base_layer,
kernel_size=convolution_kernel_size,
activation='relu',
padding='same',
kernel_initializer=initializers.he_normal(),
kernel_regularizer=regularizers.l2(weight_decay))(inputs)
encoding_layers_with_long_skip_connections.append(model)
encoding_layer_count += 1
# Encoding initialization path
model = simple_block_3d(
model,
number_of_filters_at_base_layer,
downsample=True,
convolution_kernel_size=convolution_kernel_size,
deconvolution_kernel_size=deconvolution_kernel_size,
weight_decay=weight_decay,
dropout_rate=dropout_rate)
encoding_layers_with_long_skip_connections.append(model)
encoding_layer_count += 1
# Encoding main path
number_of_bottle_neck_layers = len(bottle_neck_block_depth_schedule)
for i in range(number_of_bottle_neck_layers):
number_of_filters = number_of_filters_at_base_layer * 2**i
for j in range(bottle_neck_block_depth_schedule[i]):
do_downsample = False
if j == 0:
do_downsample = True
else:
do_downsample = False
model = bottle_neck_block_3d(
model,
number_of_filters=number_of_filters,
downsample=do_downsample,
deconvolution_kernel_size=deconvolution_kernel_size,
weight_decay=weight_decay,
dropout_rate=dropout_rate)
if j == (bottle_neck_block_depth_schedule[i] - 1):
encoding_layers_with_long_skip_connections.append(model)
encoding_layer_count += 1
encoding_layer_count -= 1
# Transition path
number_of_filters = number_of_filters_at_base_layer * 2**number_of_bottle_neck_layers
model = bottle_neck_block_3d(
model,
number_of_filters=number_of_filters,
downsample=True,
deconvolution_kernel_size=deconvolution_kernel_size,
weight_decay=weight_decay,
dropout_rate=dropout_rate)
model = bottle_neck_block_3d(
model,
number_of_filters=number_of_filters,
upsample=True,
deconvolution_kernel_size=deconvolution_kernel_size,
weight_decay=weight_decay,
dropout_rate=dropout_rate)
# Decoding main path
number_of_bottle_neck_layers = len(bottle_neck_block_depth_schedule)
for i in range(number_of_bottle_neck_layers):
number_of_filters = (number_of_filters_at_base_layer *
2**(number_of_bottle_neck_layers - i - 1))
for j in range(
bottle_neck_block_depth_schedule[number_of_bottle_neck_layers -
i - 1]):
do_upsample = False
if j == bottle_neck_block_depth_schedule[
number_of_bottle_neck_layers - i - 1] - 1:
do_upsample = True
else:
do_upsample = False
model = bottle_neck_block_3d(
model,
number_of_filters=number_of_filters,
upsample=do_upsample,
deconvolution_kernel_size=deconvolution_kernel_size,
weight_decay=weight_decay,
dropout_rate=dropout_rate)
if j == 0:
model = Conv3D(filters=(number_of_filters * 4),
kernel_size=(1, 1, 1),
padding='same')(model)
model = skip_connection(
encoding_layers_with_long_skip_connections[
encoding_layer_count - 1], model)
encoding_layer_count -= 1
# Decoding initialization path
model = simple_block_3d(
model,
number_of_filters_at_base_layer,
upsample=True,
convolution_kernel_size=convolution_kernel_size,
deconvolution_kernel_size=deconvolution_kernel_size,
weight_decay=weight_decay,
dropout_rate=dropout_rate)
# Postprocessing layer
model = Conv3D(filters=number_of_filters_at_base_layer,
kernel_size=convolution_kernel_size,
activation='relu',
padding='same',
kernel_initializer=initializers.he_normal(),
kernel_regularizer=regularizers.l2(weight_decay))(model)
encoding_layer_count -= 1
model = skip_connection(
encoding_layers_with_long_skip_connections[encoding_layer_count - 1],
model)
model = BatchNormalization()(model)
model = ThresholdedReLU(theta=0)(model)
convActivation = ''
if mode == 'classification':
if number_of_outputs == 2:
convActivation = 'sigmoid'
else:
convActivation = 'softmax'
elif mode == 'regression':
convActivation = 'linear'
else:
raise ValueError(
'mode must be either `classification` or `regression`.')
outputs = Conv3D(filters=number_of_outputs,
kernel_size=(1, 1, 1),
activation=convActivation,
kernel_regularizer=regularizers.l2(weight_decay))(model)
resunet_model = Model(inputs=inputs, outputs=outputs)
return resunet_model
def _conv_block(self,
inputs,
filters,
alpha,
kernel=(3, 3),
strides=(1, 1)):
"""Adds an initial convolution layer (with batch normalization and relu6).
# Arguments
inputs: Input tensor of shape `(rows, cols, 3)`
(with `channels_last` data format) or
(3, rows, cols) (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 32.
E.g. `(224, 224, 3)` would be one valid value.
filters: Integer, the dimensionality of the output space
(i.e. the number of output filters in the convolution).
alpha: controls the width of the network.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
kernel: An integer or tuple/list of 2 integers, specifying the
width and height of the 2D convolution window.
Can be a single integer to specify the same value for
all spatial dimensions.
strides: An integer or tuple/list of 2 integers,
specifying the strides of the convolution along the width and height.
Can be a single integer to specify the same value for
all spatial dimensions.
Specifying any stride value != 1 is incompatible with specifying
any `dilation_rate` value != 1.
# Input shape
4D tensor with shape:
`(samples, channels, rows, cols)` if data_format='channels_first'
or 4D tensor with shape:
`(samples, rows, cols, channels)` if data_format='channels_last'.
# Output shape
4D tensor with shape:
`(samples, filters, new_rows, new_cols)` if data_format='channels_first'
or 4D tensor with shape:
`(samples, new_rows, new_cols, filters)` if data_format='channels_last'.
`rows` and `cols` values might have changed due to stride.
# Returns
Output tensor of block.
"""
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
filters = int(filters * alpha)
x = ZeroPadding2D(padding=(1, 1), name='conv1_pad')(inputs)
x = Conv2D(filters,
kernel,
padding='valid',
use_bias=False,
strides=strides,
name='conv1')(x)
x = BatchNormalization(axis=channel_axis, name='conv1_bn')(x)
return ThresholdedReLU(0)(x) # 原实现relu6
def load_model(input_shape, num_labels, axis=-1, base_filter=32, depth_size=4, se_res_block=True, se_ratio=16,
noise=0.1, last_relu=False, atten_gate=False):
def conv3d(layer_input, filters, axis=-1, se_res_block=True, se_ratio=16, down_sizing=True):
if down_sizing == True:
layer_input = MaxPooling3D(pool_size=(2, 2, 2))(layer_input)
d = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(layer_input)
d = InstanceNormalization(axis=axis)(d)
d = LeakyReLU(alpha=0.3)(d)
d = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(d)
d = InstanceNormalization(axis=axis)(d)
if se_res_block == True:
se = GlobalAveragePooling3D()(d)
se = Dense(filters // se_ratio, activation='relu')(se)
se = Dense(filters, activation='sigmoid')(se)
se = Reshape([1, 1, 1, filters])(se)
d = Multiply()([d, se])
shortcut = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(layer_input)
shortcut = InstanceNormalization(axis=axis)(shortcut)
d = add([d, shortcut])
d = LeakyReLU(alpha=0.3)(d)
return d
def deconv3d(layer_input, skip_input, filters, axis=-1, se_res_block=True, se_ratio=16, atten_gate=False):
if atten_gate == True:
gating = Conv3D(filters, (1, 1, 1), use_bias=False, padding='same')(layer_input)
gating = InstanceNormalization(axis=axis)(gating)
attention = Conv3D(filters, (2, 2, 2), strides=(2, 2, 2), use_bias=False, padding='valid')(skip_input)
attention = InstanceNormalization(axis=axis)(attention)
attention = add([gating, attention])
attention = Conv3D(1, (1, 1, 1), use_bias=False, padding='same', activation='sigmoid')(attention)
# attention = Lambda(resize_by_axis, arguments={'dim_1':skip_input.get_shape().as_list()[1],'dim_2':skip_input.get_shape().as_list()[2],'ax':3})(attention) # error when None dimension is feeded.
# attention = Lambda(resize_by_axis, arguments={'dim_1':skip_input.get_shape().as_list()[1],'dim_2':skip_input.get_shape().as_list()[3],'ax':2})(attention)
attention = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(attention)
attention = UpSampling3D((2, 2, 2))(attention)
attention = CropToConcat3D(mode='crop')([attention, skip_input])
attention = Lambda(lambda x: K.tile(x, [1, 1, 1, 1, filters]))(attention)
skip_input = multiply([skip_input, attention])
u1 = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(layer_input)
u1 = Conv3DTranspose(filters, (2, 2, 2), strides=(2, 2, 2), use_bias=False, padding='same')(u1)
u1 = InstanceNormalization(axis=axis)(u1)
u1 = LeakyReLU(alpha=0.3)(u1)
u1 = CropToConcat3D()([u1, skip_input])
u2 = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u1)
u2 = InstanceNormalization(axis=axis)(u2)
u2 = LeakyReLU(alpha=0.3)(u2)
u2 = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u2)
u2 = InstanceNormalization(axis=axis)(u2)
if se_res_block == True:
se = GlobalAveragePooling3D()(u2)
se = Dense(filters // se_ratio, activation='relu')(se)
se = Dense(filters, activation='sigmoid')(se)
se = Reshape([1, 1, 1, filters])(se)
u2 = Multiply()([u2, se])
shortcut = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u1)
shortcut = InstanceNormalization(axis=axis)(shortcut)
u2 = add([u2, shortcut])
u2 = LeakyReLU(alpha=0.3)(u2)
return u2
def CropToConcat3D(mode='concat'):
def crop_to_concat_3D(concat_layers, axis=-1):
bigger_input, smaller_input = concat_layers
bigger_shape, smaller_shape = tf.shape(bigger_input), \
tf.shape(smaller_input)
sh, sw, sd = smaller_shape[1], smaller_shape[2], smaller_shape[3]
bh, bw, bd = bigger_shape[1], bigger_shape[2], bigger_shape[3]
dh, dw, dd = bh - sh, bw - sw, bd - sd
cropped_to_smaller_input = bigger_input[:, :-dh,
:-dw,
:-dd, :]
if mode == 'concat':
return K.concatenate([smaller_input, cropped_to_smaller_input], axis=axis)
elif mode == 'add':
return smaller_input + cropped_to_smaller_input
elif mode == 'crop':
return cropped_to_smaller_input
return Lambda(crop_to_concat_3D)
def resize_by_axis(image, dim_1, dim_2, ax): # it is available only for 1 channel 3D
resized_list = []
unstack_img_depth_list = tf.unstack(image, axis=ax)
for i in unstack_img_depth_list:
resized_list.append(tf.image.resize_images(i, [dim_1, dim_2])) # defaults to ResizeMethod.BILINEAR
stack_img = tf.stack(resized_list, axis=ax + 1)
return stack_img
input_img = Input(shape=input_shape, name='Input')
d0 = GaussianNoise(noise)(input_img)
d1 = Conv3D(base_filter, (3, 3, 3), use_bias=False, padding='same')(d0)
d1 = InstanceNormalization(axis=axis)(d1)
d1 = LeakyReLU(alpha=0.3)(d1)
d2 = conv3d(d1, base_filter * 2, se_res_block=se_res_block)
d3 = conv3d(d2, base_filter * 4, se_res_block=se_res_block)
d4 = conv3d(d3, base_filter * 8, se_res_block=se_res_block)
if depth_size == 4:
d5 = conv3d(d4, base_filter * 16, se_res_block=se_res_block)
u4 = deconv3d(d5, d4, base_filter * 8, se_res_block=se_res_block, atten_gate=atten_gate)
u3 = deconv3d(u4, d3, base_filter * 4, se_res_block=se_res_block, atten_gate=atten_gate)
elif depth_size == 3:
u3 = deconv3d(d4, d3, base_filter * 4, se_res_block=se_res_block, atten_gate=atten_gate)
else:
raise Exception('depth size must be 3 or 4. you put ', depth_size)
u2 = deconv3d(u3, d2, base_filter * 2, se_res_block=se_res_block, atten_gate=atten_gate)
u1 = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(u2)
u1 = Conv3DTranspose(base_filter, (2, 2, 2), strides=(2, 2, 2), use_bias=False, padding='same')(u1)
u1 = InstanceNormalization(axis=axis)(u1)
u1 = LeakyReLU(alpha=0.3)(u1)
u1 = CropToConcat3D()([u1, d1])
u1 = Conv3D(base_filter, (3, 3, 3), use_bias=False, padding='same')(u1)
u1 = InstanceNormalization(axis=axis)(u1)
u1 = LeakyReLU(alpha=0.3)(u1)
output_img = Conv3D(num_labels, kernel_size=1, strides=1, padding='same', activation='sigmoid')(u1)
if last_relu == True:
output_img = ThresholdedReLU(theta=0.5)(output_img)
model = Model(inputs=input_img, outputs=output_img)
return model
def _depthwise_conv_block(self,
inputs,
pointwise_conv_filters,
alpha,
depth_multiplier=1,
strides=(1, 1),
block_id=1):
"""Adds a depthwise convolution block.
A depthwise convolution block consists of a depthwise conv,
batch normalization, relu6, pointwise convolution,
batch normalization and relu6 activation.
# Arguments
inputs: Input tensor of shape `(rows, cols, channels)`
(with `channels_last` data format) or
(channels, rows, cols) (with `channels_first` data format).
pointwise_conv_filters: Integer, the dimensionality of the output space
(i.e. the number of output filters in the pointwise convolution).
alpha: controls the width of the network.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
depth_multiplier: The number of depthwise convolution output channels
for each input channel.
The total number of depthwise convolution output
channels will be equal to `filters_in * depth_multiplier`.
strides: An integer or tuple/list of 2 integers,
specifying the strides of the convolution along the width and height.
Can be a single integer to specify the same value for
all spatial dimensions.
Specifying any stride value != 1 is incompatible with specifying
any `dilation_rate` value != 1.
block_id: Integer, a unique identification designating the block number.
# Input shape
4D tensor with shape:
`(batch, channels, rows, cols)` if data_format='channels_first'
or 4D tensor with shape:
`(batch, rows, cols, channels)` if data_format='channels_last'.
# Output shape
4D tensor with shape:
`(batch, filters, new_rows, new_cols)` if data_format='channels_first'
or 4D tensor with shape:
`(batch, new_rows, new_cols, filters)` if data_format='channels_last'.
`rows` and `cols` values might have changed due to stride.
# Returns
Output tensor of block.
"""
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
pointwise_conv_filters = int(pointwise_conv_filters * alpha)
x = ZeroPadding2D(padding=(1, 1),
name='conv_pad_%d' % block_id)(inputs)
x = DepthwiseConv2D((3, 3),
padding='valid',
depth_multiplier=depth_multiplier,
strides=strides,
use_bias=False,
name='conv_dw_%d' % block_id)(x)
x = BatchNormalization(axis=channel_axis,
name='conv_dw_%d_bn' % block_id)(x)
x = ThresholdedReLU(0)(x) # 原实现relu6
x = Conv2D(pointwise_conv_filters, (1, 1),
padding='same',
use_bias=False,
strides=(1, 1),
name='conv_pw_%d' % block_id)(x)
x = BatchNormalization(axis=channel_axis,
name='conv_pw_%d_bn' % block_id)(x)
return ThresholdedReLU(0)(x) # 原实现relu6
def run_CNN(use_pfms=True):
batch_size = 256
epochs = 80
patience = 5
model = Sequential()
model.add(
Conv2D(filters=32,
kernel_size=(4, 20),
padding="valid",
input_shape=(1, 4, 3000)))
model.add(MaxPooling2D(pool_size=(1, 4)))
model.add(ThresholdedReLU(theta=3.0))
model.add(AveragePooling2D(pool_size=(1, 20), strides=(1, 2)))
# connect to CNN
model.add(
Conv2D(filters=32,
kernel_size=(1, 8),
strides=(1, 1),
padding="valid",
activation='relu'))
model.add(MaxPooling2D(pool_size=(1, 4)))
model.add(Dropout(0.1))
model.add(
Conv2D(filters=32,
kernel_size=(1, 8),
strides=(1, 1),
padding="valid",
activation='relu'))
model.add(MaxPooling2D(pool_size=(1, 4)))
model.add(Dropout(0.1))
model.add(
Conv2D(filters=32,
kernel_size=(1, 8),
strides=(1, 1),
padding="valid",
activation='relu'))
model.add(Flatten())
model.add(Dense(units=32, activation='relu'))
model.add(Dense(units=1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc'])
if use_pfms:
conv_weights = generate_motif_weights(model, pfmfns)
model.layers[0].set_weights(conv_weights)
# output name
outdir = args.out
if not os.path.exists(outdir): os.makedirs(outdir)
fns = [int(fn.split('_')[0]) for fn in os.listdir(outdir)]
if fns:
outfn_base = outdir + '/%d_' % (max(fns) + 1)
else:
outfn_base = outdir + '/1_'
print "output base: ", outfn_base
# save model struct
model_struct = model.to_json()
outfn_struct = open(outfn_base + 'model_struct.json', 'w+')
json.dump(model_struct, outfn_struct)
outfn_struct.close()
# dead relu detector
# outfn_deadrelu = outfn_base + 'deadrelu.csv'
# best model
outfn_bestmodel = outfn_base + 'model.h5'
# training log
outfn_log = outfn_base + 'training_log.csv'
# epoch time
outfn_epoch = outfn_base + 'epoch_time.csv'
# train CNN
hist, batch_loss_hist = train(
model,
X_train,
y_train,
X_valid,
y_valid,
batch_size,
epochs,
class_weight=False,
loss_per_batch=True,
early_stopping_metric='val_loss',
value_cutoff=2.1,
patience=patience,
min_delta=0.00001,
# deadrelu_filepath=outfn_deadrelu,
# only_first_epoch=True,
best_model_filepath=outfn_bestmodel,
log_filepath=outfn_log,
epochtime_filepath=outfn_epoch)
# training history plot
fig = CNN.plot_training_hist(hist, batch_loss_hist)
plt.savefig(outfn_base + 'training_history.pdf')
# model parameters file
outfn_params = open(outfn_base + 'params.txt', 'w+')
pred_auc = model.predict(X_valid, batch_size=128)
auc = roc_auc_score(y_valid, pred_auc)
params_paragraph = "Validation auc:%s\nParams:" % (auc)
# for key, value in params.iteritems():
# params_paragraph += "\n%s:%s"%(key,value)
# outfn_params.write(params_paragraph)
outfn_params.close()