def build_ct_net(self):
"""
可用的initialization方法：random_normal(stddev=0.0001), Orthogonal(), glorot_uniform(), lecun_uniform()
:return:
"""
model = self.model = Sequential()
print("initial shape {0}".format((3,) + self.size))
dr1 = 0.2
initial_dict = {'orthogonal': initializers.Orthogonal(), 'he_n': "he_normal"}
model.add(Lambda(vgg_preprocess, input_shape=(3,) + self.size))
model = self.conv2d_bn(model, 32, 3, 3, initial_dict['orthogonal'])
model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)))
model = self.conv2d_bn(model, 32, 3, 3, initial_dict['orthogonal'])
model.add(AveragePooling2D(pool_size=(3, 3), strides=(2, 2)))
model = self.conv2d_bn(model, 64, 3, 3, initial_dict['orthogonal'])
model.add(AveragePooling2D(pool_size=(3, 3), strides=(2, 2)))
# print(model.summary())
model.add(Flatten())
model.add(Dense(64, activation='relu', kernel_initializer=initializers.Orthogonal()))
model.add(Dropout(dr1))
# model.add(BatchNormalization())
model.add(Dense(self.n_classes, activation='softmax', kernel_initializer=initializers.Orthogonal()))
print(model.summary())
# if ft:
# self.finetune()
self.compile()
def build_textcnn(self):
"""
可用的initialization方法：random_normal(stddev=0.0001), Orthogonal(), glorot_uniform(), lecun_uniform()
:return:
"""
model = self.model = Sequential()
initial_dict = {
'orthogonal': initializers.Orthogonal(),
'he_n': "he_normal"
}
dr = 0.2
model.add(
Embedding(output_dim=64,
input_length=self.max_sequence_length,
input_dim=self.max_nb_words,
embeddings_initializer=initial_dict['he_n'],
trainable=True))
model.add(Conv1D(256, 3, kernel_initializer=initial_dict['he_n']))
model.add(Activation('relu'))
model.add(Flatten())
model.add(
Dense(64,
activation='relu',
kernel_initializer=initial_dict['he_n']))
model.add(Dropout(dr))
model.add(
Dense(self.n_classes,
activation='softmax',
kernel_initializer=initial_dict['he_n']))
print(model.summary())
self.compile()
def creat_model(input_shape, num_class):
init = initializers.Orthogonal(gain=args.norm)
sequence_input = Input(shape=input_shape)
mask = Masking(mask_value=0.)(sequence_input)
if args.aug:
mask = augmentaion()(mask)
X = Noise(0.075)(mask)
if args.model[0:2] == 'VA':
# VA
trans = Bidirectional(
LSTM(args.nhid,
recurrent_activation='sigmoid',
return_sequences=True,
implementation=2,
recurrent_initializer=init))(X)
trans = Dropout(0.5)(trans)
trans = TimeDistributed(Dense(3, kernel_initializer='zeros'))(trans)
rot = Bidirectional(
LSTM(args.nhid,
recurrent_activation='sigmoid',
return_sequences=True,
implementation=2,
recurrent_initializer=init))(X)
rot = Dropout(0.5)(rot)
rot = TimeDistributed(Dense(3, kernel_initializer='zeros'))(rot)
transform = Concatenate()([rot, trans])
X = VA()([mask, transform])
X = LSTM(args.nhid,
recurrent_activation='sigmoid',
return_sequences=True,
implementation=2,
recurrent_initializer=init)(X)
X = Dropout(0.5)(X)
X = LSTM(args.nhid,
recurrent_activation='sigmoid',
return_sequences=True,
implementation=2,
recurrent_initializer=init)(X)
X = Dropout(0.5)(X)
X = LSTM(args.nhid,
recurrent_activation='sigmoid',
return_sequences=True,
implementation=2,
recurrent_initializer=init)(X)
X = Dropout(0.5)(X)
X = Attention2()(X)
X = TimeDistributed(Dense(num_class))(X)
X = MeanOverTime()(X)
X = Dense(num_class)(X)
X = Activation('softmax')(X)
model = Model(sequence_input, X)
print(model.summary())
return model
def __init__(self, eps_std=0.05, seed=None, init=False):
# Convolutional Aware Initialization takes a long time.
# Keras model loading loads a model, performs initialization and then
# loads weights, which is an unnecessary waste of time.
# init defaults to False so that this is bypassed when loading a saved model
# passing zeros
self._init = init
self.eps_std = eps_std
self.seed = seed
self.orthogonal = initializers.Orthogonal()
self.he_uniform = initializers.he_uniform()
def get_model():
model = Sequential()
# Model parameters
rows, cols = 28, 28
input_shape = (rows, cols, 1)
nb_classes = 10
#CNN ->0.87 with 20 epoches, 102,106 parameters
model.add(
Conv2D(32,
kernel_size=(3, 3),
activation='relu',
input_shape=(input_shape),
padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(BatchNormalization())
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu',
padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(BatchNormalization())
#initializers.random_normal(stddev=0.01),kernel_initializer='random_uniform'
model.add(
Dense(128,
activation='relu',
kernel_initializer=initializers.Orthogonal(gain=5.0),
kernel_regularizer=regularizers.l2(0.01)))
model.add(Dropout(0.5))
model.add(
Dense(nb_classes,
activation='softmax',
kernel_initializer=initializers.Orthogonal(gain=5.0),
kernel_regularizer=regularizers.l2(0.01)))
print(model.summary())
return model
def __init__(self, input_dim, output_dim, **kwargs):
self.input_dim = input_dim
self.output_dim = output_dim
super(my_attention, self).__init__(**kwargs)
# def build(self, input_shape):
#为该层创建一个可训练的权重
# 创建一个可训练的权重变量矩阵
self.kernel = self.add_weight(name='kernel',
shape=(self.input_dim, self.output_dim),
initializer=initializers.Orthogonal(
gain=1.0, seed=None),
trainable=True)
def getInitializer(init_name, learning_rate, opt, functions):
if init_name == "rnormal":
init = initializers.RandomNormal()
elif init_name == "runiform":
init = initializers.RandomUniform()
elif init_name == "varscaling":
init = initializers.VarianceScaling()
elif init_name == "orth":
init = initializers.Orthogonal()
elif init_name == "id":
init = initializers.Identity()
elif init_name == "lecun_uniform":
init = initializers.lecun_uniform()
elif init_name == "glorot_normal":
init = initializers.glorot_normal()
elif init_name == "glorot_uniform":
init = initializers.glorot_uniform()
elif init_name == "he_normal":
init = initializers.he_normal()
elif init_name == "he_uniform":
init = initializers.he_uniform()
if opt == "Adam":
optimizer = optimizers.Adam(lr=learning_rate)
elif opt == "Adagrad":
optimizer = optimizers.Adagrad(lr=learning_rate)
elif opt == "Adadelta":
optimizer = optimizers.Adadelta(lr=learning_rate)
elif opt == "Adamax":
optimizer = optimizers.Adamax(lr=learning_rate)
elif opt == "Nadam":
optimizer = optimizers.Nadam(lr=learning_rate)
elif opt == "sgd":
optimizer = optimizers.SGD(lr=learning_rate)
elif opt == "RMSprop":
optimizer = optimizers.RMSprop(lr=learning_rate)
if functions.startswith("maxout"):
functions, maxout_k = functions.split("-")
maxout_k = int(maxout_k)
else:
maxout_k = 3
if functions.startswith("leakyrelu"):
if "-" in functions:
functions, maxout_k = functions.split("-")
maxout_k = float(maxout_k)
else:
maxout_k = 0.01
return init, optimizer, functions, maxout_k
def get_generative(input_dim=32,
dense_dim=200,
out_dim=50,
activation='tanh',
lr=1e-3,
activity_l1=0.0,
flag_orth_reg=False,
flag_orth_init=False,
flag_SN=False,
kernel_l2=0.0):
G_in = Input([input_dim])
if flag_orth_reg:
regu = orth_reg
else:
regu = regularizers.l2(kernel_l2)
activity_regu = regularizers.l1(activity_l1)
if flag_orth_init:
init = initializers.Orthogonal()
else:
init = 'glorot_uniform'
if flag_SN:
Dense_final = DenseSN
else:
Dense_final = Dense
x = Dense_final(out_dim,
activation='linear',
activity_regularizer=activity_regu,
kernel_regularizer=regu,
kernel_initializer=init)(G_in)
if activation == 'swish':
G_out = swish(x)
else:
G_out = Activation(activation)(x)
G = Model(G_in, G_out)
opt = Adam(lr=lr, beta_1=0.5, beta_2=0.999) #SGD(lr=lr)
G.compile(loss='mse', optimizer=opt)
return G, G_out
def get_discriminative(input_dim=32,
out_dim=50,
activation='sigmoid',
lr=1e-3,
kernel_l1=0.0,
kernel_l2=0.0,
flag_norm1=False,
flag_orth_reg=False,
flag_orth_init=False):
D_in = Input([input_dim])
if kernel_l1 > 0.0:
regu = regularizers.l1(kernel_l1)
else:
regu = regularizers.l2(kernel_l2)
if flag_orth_reg:
regu = orth_reg
if flag_orth_init:
init = initializers.Orthogonal()
else:
init = 'glorot_uniform'
if flag_norm1:
x = Lambda(lambda x: K.l2_normalize(x, axis=1))(D_in)
D_out = Dense(out_dim,
activation=activation,
kernel_regularizer=regu,
kernel_initializer=init)(x)
else:
D_out = Dense(out_dim,
activation=activation,
kernel_regularizer=regu,
kernel_initializer=init)(D_in)
D = Model(D_in, D_out)
dopt = Adam(lr=lr, beta_1=0.5, beta_2=0.999)
D.compile(loss='mse', optimizer=dopt)
return D, D_out
def __init__(self, eps_std=0.05, seed=None, init=False):
self._init = init
self.eps_std = eps_std
self.seed = seed
self.orthogonal = initializers.Orthogonal()
self.he_uniform = initializers.he_uniform()
def input_model_fc_deep(hidden_dim,
input_shape,
depth,
inputs=None,
input_layer=None,
flag_addBN=False,
flag_include_zscore=False,
zscore_weights=None,
flag_outer_swish=False,
flag_nodropout=False,
flag_orth_reg=False,
kernel_l2=0.0,
flag_orth_init=False,
flag_skip_input=False,
flag_SN=False,
flag_LMH=False,
activity_l1=0.0,
flag_BN=True):
if flag_orth_reg:
regu = orth_reg
else:
regu = regularizers.l2(kernel_l2)
activity_regu = regularizers.l1(activity_l1)
if flag_orth_init:
init = initializers.Orthogonal()
else:
init = 'glorot_uniform'
if flag_SN:
Dense_final = DenseSN
else:
Dense_final = Dense
if inputs == None:
inputs = Input(input_shape, name="input")
if flag_include_zscore:
x = ZscoreLayer(weights=zscore_weights)(inputs)
if flag_skip_input:
skip_input = x
x = Dense_final(hidden_dim,
activation='linear',
kernel_regularizer=regu,
kernel_initializer=init,
activity_regularizer=activity_regu)(x)
else:
x = Dense_final(hidden_dim,
activation='linear',
kernel_regularizer=regu,
kernel_initializer=init,
activity_regularizer=activity_regu)(inputs)
if flag_skip_input:
skip_input = inputs
else:
if flag_include_zscore:
x = ZscoreLayer(weights=zscore_weights)(input_layer)
if flag_skip_input:
skip_input = x
x = Dense_final(hidden_dim,
activation='linear',
kernel_regularizer=regu,
kernel_initializer=init,
activity_regularizer=activity_regu)(x)
else:
x = Dense_final(hidden_dim,
activation='linear',
kernel_regularizer=regu,
kernel_initializer=init,
activity_regularizer=activity_regu)(input_layer)
if flag_skip_input:
skip_input = input_layer
if flag_BN:
x = BatchNormalization()(x)
if flag_outer_swish:
x0 = outer_swish(x)
else:
x0 = swish(x)
if flag_nodropout:
flat = x0
else:
flat = Dropout(1. / (depth + 1))(x0)
LMH_list = [flat]
for i in range(depth):
if flag_skip_input:
concat_x = concatenate([flat, skip_input])
x = Dense_final(hidden_dim,
activation='linear',
kernel_regularizer=regu,
kernel_initializer=init,
activity_regularizer=activity_regu)(concat_x)
else:
x = Dense_final(hidden_dim,
activation='linear',
kernel_regularizer=regu,
kernel_initializer=init,
activity_regularizer=activity_regu)(flat)
if flag_BN:
x = BatchNormalization()(x)
if flag_outer_swish:
x0 = outer_swish(x)
else:
x0 = swish(x)
if flag_nodropout:
flat0 = x0
else:
flat0 = Dropout(1. / (depth + 1))(x0)
flat = add([flat, flat0])
if i in [0, depth / 2, depth - 1]:
LMH_list.append(flat)
if flag_addBN:
flat = BatchNormalization()(flat)
if flag_LMH:
flat = concatenate(LMH_list)
# if flag_skip_input:
# flat = concatenate([flat,skip_input])
model = Model(inputs, flat)
opt = Adam(lr=1e-3, beta_1=0.5, beta_2=0.999) #SGD(lr=lr)
model.compile(loss='mse', optimizer=opt)
shape_before_flatten = flat.shape.as_list()[1:] # [1:] to skip None
shape_flatten = np.prod(
shape_before_flatten) # value of shape in the non-batch dimension
# print shape_flatten
return model, inputs, flat, shape_flatten
def __init__(self, model_name, model_variant, input_dim,
categories_output_dim, attributes_output_dim,
class_descriptions, attributes_groups_ranges_ids):
self.model_name = model_name
self.model_variant = model_variant
print('Attribute Expert - Building LAGO_Model')
inputs = Input(shape=(input_dim, ), name="Input")
############################################
# Define the layer that maps an image representation to semantic attributes.
# In the paper, this layer is f^1_W, i.e. The mapping X-->A, with parameters W.
# Its output is an estimation for Pr(a|x)
# Set bias regularizer of the keras layer according to beta hyper param
# Note that the matrix weights are regularized explicitly through the loss
# function. Therefore, no need to also set them when defining the layer
L2_coeff = UserArgs.LG_beta
bias_regularizer = regularizers.l2(L2_coeff)
semantic_embed_layer = Dense(
attributes_output_dim,
activation='sigmoid',
trainable=True,
name='attribute_predictor',
kernel_initializer=initializers.Orthogonal(
gain=UserArgs.orth_init_gain),
bias_regularizer=bias_regularizer,
)
# Connect that layer to the model graph
Pr_a_cond_x = semantic_embed_layer(inputs)
############################################
############################################
# Define the zero shot layer.
# This layer that maps semantic attributes to class prediction.
# In the paper, this layer is f^3∘f^2_{U,V}
# i.e. The mapping A-->G-->Z, with parameters U, V
# U is the class description, V is the (soft) group assignments
# Its output is an estimation for Pr(z|x)
# Define initializers for the matrices that hold the class description
def init_train_class_description(*args):
return class_descriptions['train']
def init_val_class_description(*args):
return class_descriptions['val']
def init_test_class_description(*args):
return class_descriptions['test']
# Builds a custom LAGO layer for mapping attributes to train classes
ZS_train_layer = semantic_transfer_Layer(
categories_output_dim,
init_train_class_description(),
attributes_groups_ranges_ids,
trainable=UserArgs.SG_trainable,
ZS_type=self.model_variant,
f_build=self.build_transfer_attributes_to_classes_LAGO,
name='ZS_train_layer')
# Builds a custom LAGO layer for mapping attributes to validation class
ZS_val_layer = semantic_transfer_Layer(
categories_output_dim,
init_test_class_description(),
attributes_groups_ranges_ids,
trainable=False,
ZS_type=self.model_variant,
train_layer_ref=ZS_train_layer,
f_build=self.build_transfer_attributes_to_classes_LAGO,
name='ZS_val_layer')
# Connect those layers to model graph
ctg_train_predictions = ZS_train_layer(Pr_a_cond_x)
ctg_val_predictions = ZS_val_layer(Pr_a_cond_x)
# Define the prediction heads
predictions = [ctg_train_predictions, ctg_val_predictions, Pr_a_cond_x]
# Define the Keras model
model = Model(inputs=inputs, outputs=predictions)
self.model = model
self.dragon_trainer = DragonTrainer(
self.model_name,
f"-lr={UserArgs.initial_learning_rate}_beta={UserArgs.LG_beta}"
f"_lambda={UserArgs.LG_lambda}_gain={UserArgs.SG_gain}_psi={UserArgs.SG_psi}"
)
self.loss_params = (semantic_embed_layer.kernel,
ZS_train_layer.class_descriptions.T,
ZS_train_layer.Gamma, attributes_groups_ranges_ids)
def __init__(self, eps_std=0.05, seed=None, initialized=False):
self.eps_std = eps_std
self.seed = seed
self.orthogonal = initializers.Orthogonal() # pylint:disable=no-member
self.he_uniform = initializers.he_uniform() # pylint:disable=no-member
self.initialized = initialized
def get_model(input_dim, categories_output_dim, attributes_output_dim,
class_descriptions, attributes_groups_ranges_ids):
# Build model:
print('Building ES-ZSL model')
inputs = Input(shape=(input_dim,))
h = inputs
V_layer = Dense(attributes_output_dim, activation='linear',
use_bias=False,
# trainable=False,
trainable=True,
name='semantic_embedding_layer',
kernel_initializer=initializers.Orthogonal(
gain=common_args.orth_init_gain),
)
att_predictions = V_layer(h)
def init_S_train(*args): return class_descriptions['train']
def init_S_val(*args): return class_descriptions['val']
S_train_layer = semantic_transfer_Layer(categories_output_dim,
init_S_train(),
attributes_groups_ranges_ids,
trainable=False,
ZS_type = 'ESZSL',
f_build=build_transfer_attributes_to_classes_ESZSL,
name='ZS_train_layer')
S_val_layer = semantic_transfer_Layer(categories_output_dim,
init_S_val(),
attributes_groups_ranges_ids,
trainable=False,
ZS_type = 'ESZSL',
f_build=build_transfer_attributes_to_classes_ESZSL,
name='ZS_val_layer')
ctg_S_train_predictions = S_train_layer(att_predictions)
ctg_S_val_predictions = S_val_layer(att_predictions)
predictions = [ctg_S_train_predictions, ctg_S_val_predictions,
att_predictions]
model = ESZSL_Model(inputs=inputs, outputs=predictions)
model.inputs = inputs
model.predictions = predictions
model.output_layers = [S_train_layer, S_val_layer]
model.output_layers_dict = dict(S_train_layer=S_train_layer,
S_val_layer=S_val_layer,
V_layer=V_layer)
# An option to train ESZSL with gradient steps.
# Not fully implemented here.
# In an early version, we tried it. If you also like to try it, note that
# we had to use a learning rate decay schedule in order to converge to the
# optimum of ESZSL.
model.loss_weights = [1., 0., 0.]
model.monitor, model.mon_mode = 'val_ZS_val_layer_val_acc', 'max'
def eszsl_train_loss(y_true, y_pred):
return eszsl_loss(y_true, y_pred,
inputs, V_layer.kernel,
S_train_layer.class_descriptions.T)
def zero_loss(y_true, y_pred):
return K.constant(0.)
model.loss_list = [eszsl_train_loss, zero_loss, zero_loss]
return model
def get_model(input_dim, categories_output_dim, attributes_output_dim,
class_descriptions, attributes_groups_ranges_ids):
"""
Build a Keras model for LAGO
"""
# Build model
print('Building LAGO_Model')
# Define inputs for the model graph
inputs = Input(shape=(input_dim, ))
############################################
# Define the layer that maps an image representation to semantic attributes.
# In the paper, this layer is f^1_W, i.e. The mapping X-->A, with parameters W.
# Its output is an estimation for Pr(a|x)
# Set bias regularizer of the keras layer according to beta hyper param
# Note that the matrix weights are regularized explicitly through the loss
# function. Therefore, no need to also set them when defining the layer
L2_coeff = common_args.LG_beta
bias_regularizer = keras.regularizers.l2(L2_coeff)
semantic_embed_layer = Dense(
attributes_output_dim,
activation='sigmoid',
trainable=True,
name='attribute_predictor',
kernel_initializer=initializers.Orthogonal(
gain=common_args.orth_init_gain),
bias_regularizer=bias_regularizer,
)
# Connect that layer to the model graph
Pr_a_cond_x = semantic_embed_layer(inputs)
############################################
############################################
# Define the zero shot layer.
# This layer that maps semantic attributes to class prediction.
# In the paper, this layer is f^3∘f^2_{U,V}
# i.e. The mapping A-->G-->Z, with parameters U, V
# U is the class description, V is the (soft) group assignments
# Its output is an estimation for Pr(z|x)
# Define initializers for the matrices that hold the class description
def init_train_class_description(*args):
return class_descriptions['train']
def init_val_class_description(*args):
return class_descriptions['val']
# Builds a custom LAGO layer for mapping attributes to train classes
ZS_train_layer = semantic_transfer_Layer(
categories_output_dim,
init_train_class_description(),
attributes_groups_ranges_ids,
trainable=common_args.SG_trainable,
ZS_type=common_args.model_variant,
f_build=build_transfer_attributes_to_classes_LAGO,
name='ZS_train_layer')
# Builds a custom LAGO layer for mapping attributes to validation class
ZS_val_layer = semantic_transfer_Layer(
categories_output_dim,
init_val_class_description(),
attributes_groups_ranges_ids,
trainable=False,
ZS_type=common_args.model_variant,
train_layer_ref=ZS_train_layer,
f_build=build_transfer_attributes_to_classes_LAGO,
name='ZS_val_layer')
# Connect those layers to model graph
ctg_train_predictions = ZS_train_layer(Pr_a_cond_x)
ctg_val_predictions = ZS_val_layer(Pr_a_cond_x)
# Define the prediction heads
predictions = [ctg_train_predictions, ctg_val_predictions, Pr_a_cond_x]
# Define the Keras model
model = LAGO_Model(inputs=inputs, outputs=predictions)
# Add additional (object oriented) attributes to the keras model.
# Most will be used as inputs for Keras model.compile,
# or for the alternating optimization
model.predictions = predictions
# The loss heads are only over the train class predictions and attributes
model.loss_weights = [1., 0., common_args.attributes_weight]
# Define the monitor on validation data for early-stopping.
# 'val_ZS_val_ctg_val_acc' is a metric name automatically generated by Keras
# Its meaning factors to the following:
# 'val_' indicates using samples of the validation set
# 'ZS_val_layer' indicates fetching prediction from the validation head
# of the model graph.
# 'val_acc' indicates using the accuracy metric that compares
# against the validation classes
model.monitor, model.mon_mode = 'val_ZS_val_layer_val_acc', 'max'
# Saving pointer for the layers. It is used for accessing layer activations
# during debugging.
model.output_layers_dict = dict(ZS_train_layer=ZS_train_layer,
ZS_val_layer=ZS_val_layer,
semantic_embed_layer=semantic_embed_layer)
# Define the loss
def LAGO_train_loss(y_true, y_pred):
""" Wraps the LAGO_loss according to Keras API
"""
return LAGO_loss(y_true, y_pred, semantic_embed_layer.kernel,
ZS_train_layer.class_descriptions.T,
ZS_train_layer.Gamma, attributes_groups_ranges_ids)
def zero_loss(y_true, y_pred):
""" ZERO loss (above the validation classes head) according to Keras API.
This makes sure that no computations are made
through the validation classes head.
"""
return K.constant(0.)
model.loss_list = [LAGO_train_loss, zero_loss, LAGO_attr_loss]
return model
def proposed(type, train, test, code, epoch, batch):
# Load MNIST train and test data
X_train = np.loadtxt(train, delimiter=',', dtype=None)
X_test = np.loadtxt(test, delimiter=',', dtype=None)
# z_list : define experiment code(Z) size
z_list = [code]
autoencoder = [[] for i in range(len(z_list))]
# E : epoch, BS = batch size
E = epoch
BS = batch
# Train model and save data(code(Z), output and total loss data)
model_index = 0
total_summary_loss_data = [
'model_type', 'z_size', 'train_loss', 'test_loss'
]
for z_size in z_list:
# Define models
INPUT_SIZE = 784
HIDDEN_SIZE1 = 2000
HIDDEN_SIZE2 = z_size
if type == "digit":
w_initializer = initializers.Orthogonal(gain=1.0, seed=None)
b_initializer = initializers.random_normal(mean=0.0,
stddev=0.05,
seed=None)
else:
w_initializer = initializers.RandomUniform(minval=-0.05,
maxval=0.05,
seed=None)
b_initializer = initializers.glorot_normal(seed=None)
dense1 = Input(shape=(INPUT_SIZE, ))
dense2 = Dense(HIDDEN_SIZE1,
activation='relu',
kernel_initializer=w_initializer,
bias_initializer=b_initializer)(dense1)
dense3 = Dense(HIDDEN_SIZE2,
activation='linear',
kernel_initializer=w_initializer,
bias_initializer=b_initializer)(dense2)
dense4 = Dense(HIDDEN_SIZE1,
activation='relu',
kernel_initializer=w_initializer,
bias_initializer=b_initializer)(dense3)
dense5 = Dense(INPUT_SIZE,
activation='linear',
kernel_initializer=w_initializer,
bias_initializer=b_initializer)(dense4)
autoencoder[model_index] = Model(dense1, dense5)
sgd = optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
autoencoder[model_index].compile(loss='mean_squared_error',
optimizer=sgd)
autoencoder[model_index].fit(X_train,
X_train,
epochs=E,
batch_size=BS,
verbose=0)
# Get output and calculate loss
get_output = K.function([autoencoder[model_index].layers[0].input],
[autoencoder[model_index].layers[4].output])
train_output = get_output([X_train])[0]
test_output = get_output([X_test])[0]
train_loss = np.sum((X_train - train_output)**
2) / (X_train.shape[0] * X_train.shape[1])
test_loss = np.sum(
(X_test - test_output)**2) / (X_test.shape[0] * X_test.shape[1])
summary_loss_data = ['LAE', z_size, train_loss, test_loss]
total_summary_loss_data = np.vstack(
(total_summary_loss_data, summary_loss_data))
np.savetxt("total_loss.csv",
total_summary_loss_data,
delimiter=',',
fmt='%s')
# Get code(Z)
get_z = K.function([autoencoder[model_index].layers[0].input],
[autoencoder[model_index].layers[2].output])
test_z = get_z([X_test])[0]
np.savetxt("test_code.csv", test_z, delimiter=',')
model_index = model_index + 1
# Calculate and save LAE reconstruct (* LAE reconstruct = LAE output)
for z_index in range(0, len(z_list)):
z_size = z_list[z_index]
test_recon = [[]]
get_z = K.function([autoencoder[z_index].layers[0].input],
[autoencoder[z_index].layers[2].output])
temp_Z = get_z([X_test])[0]
for data_index in range(0, 10000):
temp_vec = np.dot(
temp_Z[data_index], autoencoder[z_index].layers[3].get_weights(
)[0]) + autoencoder[z_index].layers[3].get_weights()[1]
B = []
for k in range(0, len(temp_vec)):
if (temp_vec[k] >= 0.0):
B.extend([k])
w34 = autoencoder[z_index].layers[3].get_weights()[0]
w34 = w34[:, B]
w45 = autoencoder[z_index].layers[4].get_weights()[0]
w45 = w45[B, :]
W35 = np.dot(w34, w45)
b3 = autoencoder[z_index].layers[3].get_weights()[1]
b3 = b3[B]
b4 = autoencoder[z_index].layers[4].get_weights()[1]
temp_recon = np.dot(temp_Z[data_index], W35) + np.dot(b3, w45) + b4
if (data_index == 0):
test_recon = temp_recon
else:
test_recon = np.vstack((test_recon, temp_recon))
if ((data_index + 1) % 1000 == 0):
print(str(data_index + 1) + " finish!")
np.savetxt("test_out.csv", test_recon, delimiter=',')
# Print total loss
print(total_summary_loss_data)
print("learning proposed autoencoder modol finish! \n")
def get_lstm(n_units):
return LSTM(units=n_units,
return_sequences=True,
kernel_initializer=initializers.glorot_uniform(seed=0),
recurrent_initializer=initializers.Orthogonal(seed=0))
def build_model(input_shape: tuple,
gpu,
write_result_out_dir,
pre_model=None,
freeze=False,
noise=None,
verbose=True,
savefig=True):
if gpu:
from keras.layers import CuDNNLSTM as LSTM
else:
from keras.layers import LSTM
# construct the model
input_layer = Input(input_shape)
if noise:
noise_input = GaussianNoise(np.sqrt(noise))(input_layer)
dense = TimeDistributed(
Dense(10,
kernel_regularizer=regularizers.l2(0.01),
kernel_initializer=initializers.glorot_uniform(seed=0),
bias_initializer=initializers.Zeros()))(noise_input)
else:
dense = TimeDistributed(
Dense(10,
kernel_regularizer=regularizers.l2(0.01),
kernel_initializer=initializers.glorot_uniform(seed=0),
bias_initializer=initializers.Zeros()))(input_layer)
lstm1 = LSTM(60,
return_sequences=True,
kernel_regularizer=regularizers.l2(0.01),
kernel_initializer=initializers.glorot_uniform(seed=0),
recurrent_initializer=initializers.Orthogonal(seed=0),
bias_initializer=initializers.Zeros())(dense)
lstm1 = BatchNormalization()(lstm1)
lstm2 = LSTM(60,
return_sequences=False,
kernel_regularizer=regularizers.l2(0.01),
kernel_initializer=initializers.glorot_uniform(seed=0),
recurrent_initializer=initializers.Orthogonal(seed=0),
bias_initializer=initializers.Zeros())(lstm1)
lstm2 = BatchNormalization()(lstm2)
output_layer = Dense(
1,
activation='sigmoid',
kernel_regularizer=regularizers.l2(0.01),
kernel_initializer=initializers.glorot_uniform(seed=0),
bias_initializer=initializers.Zeros())(lstm2)
model = Model(inputs=input_layer, outputs=output_layer)
if savefig:
plot_model(model,
to_file=f'{write_result_out_dir}/architecture.png',
show_shapes=True,
show_layer_names=False)
# transfer weights from pre-trained model
if pre_model:
for i in range(2, len(model.layers) - 1):
model.layers[i].set_weights(pre_model.layers[i].get_weights())
if freeze:
model.layers[i].trainable = False
model.compile(optimizer=Adam(), loss='mse', metrics=['accuracy'])
if verbose: print(model.summary())
return model
initializers.random_uniform(minval=-0.2, seed=42),
dict(class_name="random_uniform", minval=-0.2, maxval=0.05, seed=42),
id="ru_1",
),
pytest.param(
initializers.TruncatedNormal(0.1),
dict(class_name="truncated_normal", mean=0.1, stddev=0.05, seed=None),
id="tn_0",
),
pytest.param(
initializers.truncated_normal(mean=0.2, stddev=0.003, seed=42),
dict(class_name="truncated_normal", mean=0.2, stddev=0.003, seed=42),
id="tn_1",
),
pytest.param(
initializers.Orthogonal(1.1),
dict(class_name="orthogonal", gain=1.1, seed=None),
id="o_0",
),
pytest.param(
initializers.orthogonal(gain=1.2, seed=42),
dict(class_name="orthogonal", gain=1.2, seed=42),
id="o_1",
),
pytest.param(initializers.Identity(1.1), dict(class_name="identity", gain=1.1), id="i_0"),
pytest.param(initializers.identity(), dict(class_name="identity", gain=1.0), id="i_1"),
#################### VarianceScaling ####################
pytest.param(
initializers.glorot_normal(), dict(class_name="glorot_normal", seed=None), id="gn_0"
),
pytest.param(
os.environ["CUDA_VISIBLE_DEVICES"] ='0,1'
# 按需进行设置,如果只用一张显卡，可以忽略下面的multi_gpu_model(model, gpus=n)及其相关语句
if __name__ == '__main__':
Xception_model = Xception(weights=None, include_top=False, input_shape=(299,299,3),pooling='Average')
#实际上也可以设置weights='imagenet',不过这样做的话框架判断用户目录下，即~/.keras/models判断是否有对应的权重文件，
# 没有的话会执行下载，建议自己另外加载就好。具体参看http://192.168.3.126/Kwong/totem_begin/blob/master/README.md
# Xception_model = load_model('xception_weights_tf_dim_ordering_tf_kernels_notop.h5')
# 如果加载的是imageNet预训练权重，需要添加另外的预处理方法(去imageNet数据集的均值和标准差)
# 详见：http://192.168.3.126/Kwong/totem_begin/blob/master/tricks_in_processing_and_training/README.md
top_model_avg = Sequential()
top_model_avg.add(Flatten(input_shape=Xception_model.output_shape[1:],name='flatten_Average'))
#top_model_avg.add(Dropout(0.5))
top_model_avg.add(BatchNormalization())
top_model_avg.add(Dense(48, name='dense_1', kernel_initializer=initializers.Orthogonal()))
top_model_avg.add(advanced_activations.PReLU(alpha_initializer='zeros'))
#top_model_avg.add(Dropout(0.5))
top_model_avg.add(BatchNormalization())
top_model_avg.add(Dense(2, name='dense_2', activation="softmax", kernel_initializer=initializers.Orthogonal()))
model = Sequential()
model.add(Xception_model)
model.add(top_model_avg)
# 以上是使用序列式定义模型的方法
#Xception_model2 = Xception(weights=None, include_top=False, input_shape=(299,299,3),pooling='Average')
#base_out = Flatten()(Xception_model2.output)
#drout1= Dropout(0.5)(base_out)
#dense1= Dense(64,activation='relu',kernel_initializer=initializers.he_normal(seed=None))(drout1)
#drout2= Dropout(0.5)(dense1)
