def build_ct_net(self): """ 可用的initialization方法:random_normal(stddev=0.0001), Orthogonal(), glorot_uniform(), lecun_uniform() :return: """ model = self.model = Sequential() print("initial shape {0}".format((3,) + self.size)) dr1 = 0.2 initial_dict = {'orthogonal': initializers.Orthogonal(), 'he_n': "he_normal"} model.add(Lambda(vgg_preprocess, input_shape=(3,) + self.size)) model = self.conv2d_bn(model, 32, 3, 3, initial_dict['orthogonal']) model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2))) model = self.conv2d_bn(model, 32, 3, 3, initial_dict['orthogonal']) model.add(AveragePooling2D(pool_size=(3, 3), strides=(2, 2))) model = self.conv2d_bn(model, 64, 3, 3, initial_dict['orthogonal']) model.add(AveragePooling2D(pool_size=(3, 3), strides=(2, 2))) # print(model.summary()) model.add(Flatten()) model.add(Dense(64, activation='relu', kernel_initializer=initializers.Orthogonal())) model.add(Dropout(dr1)) # model.add(BatchNormalization()) model.add(Dense(self.n_classes, activation='softmax', kernel_initializer=initializers.Orthogonal())) print(model.summary()) # if ft: # self.finetune() self.compile()
def build_textcnn(self): """ 可用的initialization方法:random_normal(stddev=0.0001), Orthogonal(), glorot_uniform(), lecun_uniform() :return: """ model = self.model = Sequential() initial_dict = { 'orthogonal': initializers.Orthogonal(), 'he_n': "he_normal" } dr = 0.2 model.add( Embedding(output_dim=64, input_length=self.max_sequence_length, input_dim=self.max_nb_words, embeddings_initializer=initial_dict['he_n'], trainable=True)) model.add(Conv1D(256, 3, kernel_initializer=initial_dict['he_n'])) model.add(Activation('relu')) model.add(Flatten()) model.add( Dense(64, activation='relu', kernel_initializer=initial_dict['he_n'])) model.add(Dropout(dr)) model.add( Dense(self.n_classes, activation='softmax', kernel_initializer=initial_dict['he_n'])) print(model.summary()) self.compile()
def creat_model(input_shape, num_class): init = initializers.Orthogonal(gain=args.norm) sequence_input = Input(shape=input_shape) mask = Masking(mask_value=0.)(sequence_input) if args.aug: mask = augmentaion()(mask) X = Noise(0.075)(mask) if args.model[0:2] == 'VA': # VA trans = Bidirectional( LSTM(args.nhid, recurrent_activation='sigmoid', return_sequences=True, implementation=2, recurrent_initializer=init))(X) trans = Dropout(0.5)(trans) trans = TimeDistributed(Dense(3, kernel_initializer='zeros'))(trans) rot = Bidirectional( LSTM(args.nhid, recurrent_activation='sigmoid', return_sequences=True, implementation=2, recurrent_initializer=init))(X) rot = Dropout(0.5)(rot) rot = TimeDistributed(Dense(3, kernel_initializer='zeros'))(rot) transform = Concatenate()([rot, trans]) X = VA()([mask, transform]) X = LSTM(args.nhid, recurrent_activation='sigmoid', return_sequences=True, implementation=2, recurrent_initializer=init)(X) X = Dropout(0.5)(X) X = LSTM(args.nhid, recurrent_activation='sigmoid', return_sequences=True, implementation=2, recurrent_initializer=init)(X) X = Dropout(0.5)(X) X = LSTM(args.nhid, recurrent_activation='sigmoid', return_sequences=True, implementation=2, recurrent_initializer=init)(X) X = Dropout(0.5)(X) X = Attention2()(X) X = TimeDistributed(Dense(num_class))(X) X = MeanOverTime()(X) X = Dense(num_class)(X) X = Activation('softmax')(X) model = Model(sequence_input, X) print(model.summary()) return model
def __init__(self, eps_std=0.05, seed=None, init=False): # Convolutional Aware Initialization takes a long time. # Keras model loading loads a model, performs initialization and then # loads weights, which is an unnecessary waste of time. # init defaults to False so that this is bypassed when loading a saved model # passing zeros self._init = init self.eps_std = eps_std self.seed = seed self.orthogonal = initializers.Orthogonal() self.he_uniform = initializers.he_uniform()
def get_model(): model = Sequential() # Model parameters rows, cols = 28, 28 input_shape = (rows, cols, 1) nb_classes = 10 #CNN ->0.87 with 20 epoches, 102,106 parameters model.add( Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(input_shape), padding='same')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.5)) model.add(BatchNormalization()) model.add(Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.5)) model.add(Flatten()) model.add(BatchNormalization()) #initializers.random_normal(stddev=0.01),kernel_initializer='random_uniform' model.add( Dense(128, activation='relu', kernel_initializer=initializers.Orthogonal(gain=5.0), kernel_regularizer=regularizers.l2(0.01))) model.add(Dropout(0.5)) model.add( Dense(nb_classes, activation='softmax', kernel_initializer=initializers.Orthogonal(gain=5.0), kernel_regularizer=regularizers.l2(0.01))) print(model.summary()) return model
def __init__(self, input_dim, output_dim, **kwargs): self.input_dim = input_dim self.output_dim = output_dim super(my_attention, self).__init__(**kwargs) # def build(self, input_shape): #为该层创建一个可训练的权重 # 创建一个可训练的权重变量矩阵 self.kernel = self.add_weight(name='kernel', shape=(self.input_dim, self.output_dim), initializer=initializers.Orthogonal( gain=1.0, seed=None), trainable=True)
def getInitializer(init_name, learning_rate, opt, functions): if init_name == "rnormal": init = initializers.RandomNormal() elif init_name == "runiform": init = initializers.RandomUniform() elif init_name == "varscaling": init = initializers.VarianceScaling() elif init_name == "orth": init = initializers.Orthogonal() elif init_name == "id": init = initializers.Identity() elif init_name == "lecun_uniform": init = initializers.lecun_uniform() elif init_name == "glorot_normal": init = initializers.glorot_normal() elif init_name == "glorot_uniform": init = initializers.glorot_uniform() elif init_name == "he_normal": init = initializers.he_normal() elif init_name == "he_uniform": init = initializers.he_uniform() if opt == "Adam": optimizer = optimizers.Adam(lr=learning_rate) elif opt == "Adagrad": optimizer = optimizers.Adagrad(lr=learning_rate) elif opt == "Adadelta": optimizer = optimizers.Adadelta(lr=learning_rate) elif opt == "Adamax": optimizer = optimizers.Adamax(lr=learning_rate) elif opt == "Nadam": optimizer = optimizers.Nadam(lr=learning_rate) elif opt == "sgd": optimizer = optimizers.SGD(lr=learning_rate) elif opt == "RMSprop": optimizer = optimizers.RMSprop(lr=learning_rate) if functions.startswith("maxout"): functions, maxout_k = functions.split("-") maxout_k = int(maxout_k) else: maxout_k = 3 if functions.startswith("leakyrelu"): if "-" in functions: functions, maxout_k = functions.split("-") maxout_k = float(maxout_k) else: maxout_k = 0.01 return init, optimizer, functions, maxout_k
def get_generative(input_dim=32, dense_dim=200, out_dim=50, activation='tanh', lr=1e-3, activity_l1=0.0, flag_orth_reg=False, flag_orth_init=False, flag_SN=False, kernel_l2=0.0): G_in = Input([input_dim]) if flag_orth_reg: regu = orth_reg else: regu = regularizers.l2(kernel_l2) activity_regu = regularizers.l1(activity_l1) if flag_orth_init: init = initializers.Orthogonal() else: init = 'glorot_uniform' if flag_SN: Dense_final = DenseSN else: Dense_final = Dense x = Dense_final(out_dim, activation='linear', activity_regularizer=activity_regu, kernel_regularizer=regu, kernel_initializer=init)(G_in) if activation == 'swish': G_out = swish(x) else: G_out = Activation(activation)(x) G = Model(G_in, G_out) opt = Adam(lr=lr, beta_1=0.5, beta_2=0.999) #SGD(lr=lr) G.compile(loss='mse', optimizer=opt) return G, G_out
def get_discriminative(input_dim=32, out_dim=50, activation='sigmoid', lr=1e-3, kernel_l1=0.0, kernel_l2=0.0, flag_norm1=False, flag_orth_reg=False, flag_orth_init=False): D_in = Input([input_dim]) if kernel_l1 > 0.0: regu = regularizers.l1(kernel_l1) else: regu = regularizers.l2(kernel_l2) if flag_orth_reg: regu = orth_reg if flag_orth_init: init = initializers.Orthogonal() else: init = 'glorot_uniform' if flag_norm1: x = Lambda(lambda x: K.l2_normalize(x, axis=1))(D_in) D_out = Dense(out_dim, activation=activation, kernel_regularizer=regu, kernel_initializer=init)(x) else: D_out = Dense(out_dim, activation=activation, kernel_regularizer=regu, kernel_initializer=init)(D_in) D = Model(D_in, D_out) dopt = Adam(lr=lr, beta_1=0.5, beta_2=0.999) D.compile(loss='mse', optimizer=dopt) return D, D_out
def __init__(self, eps_std=0.05, seed=None, init=False): self._init = init self.eps_std = eps_std self.seed = seed self.orthogonal = initializers.Orthogonal() self.he_uniform = initializers.he_uniform()
def input_model_fc_deep(hidden_dim, input_shape, depth, inputs=None, input_layer=None, flag_addBN=False, flag_include_zscore=False, zscore_weights=None, flag_outer_swish=False, flag_nodropout=False, flag_orth_reg=False, kernel_l2=0.0, flag_orth_init=False, flag_skip_input=False, flag_SN=False, flag_LMH=False, activity_l1=0.0, flag_BN=True): if flag_orth_reg: regu = orth_reg else: regu = regularizers.l2(kernel_l2) activity_regu = regularizers.l1(activity_l1) if flag_orth_init: init = initializers.Orthogonal() else: init = 'glorot_uniform' if flag_SN: Dense_final = DenseSN else: Dense_final = Dense if inputs == None: inputs = Input(input_shape, name="input") if flag_include_zscore: x = ZscoreLayer(weights=zscore_weights)(inputs) if flag_skip_input: skip_input = x x = Dense_final(hidden_dim, activation='linear', kernel_regularizer=regu, kernel_initializer=init, activity_regularizer=activity_regu)(x) else: x = Dense_final(hidden_dim, activation='linear', kernel_regularizer=regu, kernel_initializer=init, activity_regularizer=activity_regu)(inputs) if flag_skip_input: skip_input = inputs else: if flag_include_zscore: x = ZscoreLayer(weights=zscore_weights)(input_layer) if flag_skip_input: skip_input = x x = Dense_final(hidden_dim, activation='linear', kernel_regularizer=regu, kernel_initializer=init, activity_regularizer=activity_regu)(x) else: x = Dense_final(hidden_dim, activation='linear', kernel_regularizer=regu, kernel_initializer=init, activity_regularizer=activity_regu)(input_layer) if flag_skip_input: skip_input = input_layer if flag_BN: x = BatchNormalization()(x) if flag_outer_swish: x0 = outer_swish(x) else: x0 = swish(x) if flag_nodropout: flat = x0 else: flat = Dropout(1. / (depth + 1))(x0) LMH_list = [flat] for i in range(depth): if flag_skip_input: concat_x = concatenate([flat, skip_input]) x = Dense_final(hidden_dim, activation='linear', kernel_regularizer=regu, kernel_initializer=init, activity_regularizer=activity_regu)(concat_x) else: x = Dense_final(hidden_dim, activation='linear', kernel_regularizer=regu, kernel_initializer=init, activity_regularizer=activity_regu)(flat) if flag_BN: x = BatchNormalization()(x) if flag_outer_swish: x0 = outer_swish(x) else: x0 = swish(x) if flag_nodropout: flat0 = x0 else: flat0 = Dropout(1. / (depth + 1))(x0) flat = add([flat, flat0]) if i in [0, depth / 2, depth - 1]: LMH_list.append(flat) if flag_addBN: flat = BatchNormalization()(flat) if flag_LMH: flat = concatenate(LMH_list) # if flag_skip_input: # flat = concatenate([flat,skip_input]) model = Model(inputs, flat) opt = Adam(lr=1e-3, beta_1=0.5, beta_2=0.999) #SGD(lr=lr) model.compile(loss='mse', optimizer=opt) shape_before_flatten = flat.shape.as_list()[1:] # [1:] to skip None shape_flatten = np.prod( shape_before_flatten) # value of shape in the non-batch dimension # print shape_flatten return model, inputs, flat, shape_flatten
def __init__(self, model_name, model_variant, input_dim, categories_output_dim, attributes_output_dim, class_descriptions, attributes_groups_ranges_ids): self.model_name = model_name self.model_variant = model_variant print('Attribute Expert - Building LAGO_Model') inputs = Input(shape=(input_dim, ), name="Input") ############################################ # Define the layer that maps an image representation to semantic attributes. # In the paper, this layer is f^1_W, i.e. The mapping X-->A, with parameters W. # Its output is an estimation for Pr(a|x) # Set bias regularizer of the keras layer according to beta hyper param # Note that the matrix weights are regularized explicitly through the loss # function. Therefore, no need to also set them when defining the layer L2_coeff = UserArgs.LG_beta bias_regularizer = regularizers.l2(L2_coeff) semantic_embed_layer = Dense( attributes_output_dim, activation='sigmoid', trainable=True, name='attribute_predictor', kernel_initializer=initializers.Orthogonal( gain=UserArgs.orth_init_gain), bias_regularizer=bias_regularizer, ) # Connect that layer to the model graph Pr_a_cond_x = semantic_embed_layer(inputs) ############################################ ############################################ # Define the zero shot layer. # This layer that maps semantic attributes to class prediction. # In the paper, this layer is f^3∘f^2_{U,V} # i.e. The mapping A-->G-->Z, with parameters U, V # U is the class description, V is the (soft) group assignments # Its output is an estimation for Pr(z|x) # Define initializers for the matrices that hold the class description def init_train_class_description(*args): return class_descriptions['train'] def init_val_class_description(*args): return class_descriptions['val'] def init_test_class_description(*args): return class_descriptions['test'] # Builds a custom LAGO layer for mapping attributes to train classes ZS_train_layer = semantic_transfer_Layer( categories_output_dim, init_train_class_description(), attributes_groups_ranges_ids, trainable=UserArgs.SG_trainable, ZS_type=self.model_variant, f_build=self.build_transfer_attributes_to_classes_LAGO, name='ZS_train_layer') # Builds a custom LAGO layer for mapping attributes to validation class ZS_val_layer = semantic_transfer_Layer( categories_output_dim, init_test_class_description(), attributes_groups_ranges_ids, trainable=False, ZS_type=self.model_variant, train_layer_ref=ZS_train_layer, f_build=self.build_transfer_attributes_to_classes_LAGO, name='ZS_val_layer') # Connect those layers to model graph ctg_train_predictions = ZS_train_layer(Pr_a_cond_x) ctg_val_predictions = ZS_val_layer(Pr_a_cond_x) # Define the prediction heads predictions = [ctg_train_predictions, ctg_val_predictions, Pr_a_cond_x] # Define the Keras model model = Model(inputs=inputs, outputs=predictions) self.model = model self.dragon_trainer = DragonTrainer( self.model_name, f"-lr={UserArgs.initial_learning_rate}_beta={UserArgs.LG_beta}" f"_lambda={UserArgs.LG_lambda}_gain={UserArgs.SG_gain}_psi={UserArgs.SG_psi}" ) self.loss_params = (semantic_embed_layer.kernel, ZS_train_layer.class_descriptions.T, ZS_train_layer.Gamma, attributes_groups_ranges_ids)
def __init__(self, eps_std=0.05, seed=None, initialized=False): self.eps_std = eps_std self.seed = seed self.orthogonal = initializers.Orthogonal() # pylint:disable=no-member self.he_uniform = initializers.he_uniform() # pylint:disable=no-member self.initialized = initialized
def get_model(input_dim, categories_output_dim, attributes_output_dim, class_descriptions, attributes_groups_ranges_ids): # Build model: print('Building ES-ZSL model') inputs = Input(shape=(input_dim,)) h = inputs V_layer = Dense(attributes_output_dim, activation='linear', use_bias=False, # trainable=False, trainable=True, name='semantic_embedding_layer', kernel_initializer=initializers.Orthogonal( gain=common_args.orth_init_gain), ) att_predictions = V_layer(h) def init_S_train(*args): return class_descriptions['train'] def init_S_val(*args): return class_descriptions['val'] S_train_layer = semantic_transfer_Layer(categories_output_dim, init_S_train(), attributes_groups_ranges_ids, trainable=False, ZS_type = 'ESZSL', f_build=build_transfer_attributes_to_classes_ESZSL, name='ZS_train_layer') S_val_layer = semantic_transfer_Layer(categories_output_dim, init_S_val(), attributes_groups_ranges_ids, trainable=False, ZS_type = 'ESZSL', f_build=build_transfer_attributes_to_classes_ESZSL, name='ZS_val_layer') ctg_S_train_predictions = S_train_layer(att_predictions) ctg_S_val_predictions = S_val_layer(att_predictions) predictions = [ctg_S_train_predictions, ctg_S_val_predictions, att_predictions] model = ESZSL_Model(inputs=inputs, outputs=predictions) model.inputs = inputs model.predictions = predictions model.output_layers = [S_train_layer, S_val_layer] model.output_layers_dict = dict(S_train_layer=S_train_layer, S_val_layer=S_val_layer, V_layer=V_layer) # An option to train ESZSL with gradient steps. # Not fully implemented here. # In an early version, we tried it. If you also like to try it, note that # we had to use a learning rate decay schedule in order to converge to the # optimum of ESZSL. model.loss_weights = [1., 0., 0.] model.monitor, model.mon_mode = 'val_ZS_val_layer_val_acc', 'max' def eszsl_train_loss(y_true, y_pred): return eszsl_loss(y_true, y_pred, inputs, V_layer.kernel, S_train_layer.class_descriptions.T) def zero_loss(y_true, y_pred): return K.constant(0.) model.loss_list = [eszsl_train_loss, zero_loss, zero_loss] return model
def get_model(input_dim, categories_output_dim, attributes_output_dim, class_descriptions, attributes_groups_ranges_ids): """ Build a Keras model for LAGO """ # Build model print('Building LAGO_Model') # Define inputs for the model graph inputs = Input(shape=(input_dim, )) ############################################ # Define the layer that maps an image representation to semantic attributes. # In the paper, this layer is f^1_W, i.e. The mapping X-->A, with parameters W. # Its output is an estimation for Pr(a|x) # Set bias regularizer of the keras layer according to beta hyper param # Note that the matrix weights are regularized explicitly through the loss # function. Therefore, no need to also set them when defining the layer L2_coeff = common_args.LG_beta bias_regularizer = keras.regularizers.l2(L2_coeff) semantic_embed_layer = Dense( attributes_output_dim, activation='sigmoid', trainable=True, name='attribute_predictor', kernel_initializer=initializers.Orthogonal( gain=common_args.orth_init_gain), bias_regularizer=bias_regularizer, ) # Connect that layer to the model graph Pr_a_cond_x = semantic_embed_layer(inputs) ############################################ ############################################ # Define the zero shot layer. # This layer that maps semantic attributes to class prediction. # In the paper, this layer is f^3∘f^2_{U,V} # i.e. The mapping A-->G-->Z, with parameters U, V # U is the class description, V is the (soft) group assignments # Its output is an estimation for Pr(z|x) # Define initializers for the matrices that hold the class description def init_train_class_description(*args): return class_descriptions['train'] def init_val_class_description(*args): return class_descriptions['val'] # Builds a custom LAGO layer for mapping attributes to train classes ZS_train_layer = semantic_transfer_Layer( categories_output_dim, init_train_class_description(), attributes_groups_ranges_ids, trainable=common_args.SG_trainable, ZS_type=common_args.model_variant, f_build=build_transfer_attributes_to_classes_LAGO, name='ZS_train_layer') # Builds a custom LAGO layer for mapping attributes to validation class ZS_val_layer = semantic_transfer_Layer( categories_output_dim, init_val_class_description(), attributes_groups_ranges_ids, trainable=False, ZS_type=common_args.model_variant, train_layer_ref=ZS_train_layer, f_build=build_transfer_attributes_to_classes_LAGO, name='ZS_val_layer') # Connect those layers to model graph ctg_train_predictions = ZS_train_layer(Pr_a_cond_x) ctg_val_predictions = ZS_val_layer(Pr_a_cond_x) # Define the prediction heads predictions = [ctg_train_predictions, ctg_val_predictions, Pr_a_cond_x] # Define the Keras model model = LAGO_Model(inputs=inputs, outputs=predictions) # Add additional (object oriented) attributes to the keras model. # Most will be used as inputs for Keras model.compile, # or for the alternating optimization model.predictions = predictions # The loss heads are only over the train class predictions and attributes model.loss_weights = [1., 0., common_args.attributes_weight] # Define the monitor on validation data for early-stopping. # 'val_ZS_val_ctg_val_acc' is a metric name automatically generated by Keras # Its meaning factors to the following: # 'val_' indicates using samples of the validation set # 'ZS_val_layer' indicates fetching prediction from the validation head # of the model graph. # 'val_acc' indicates using the accuracy metric that compares # against the validation classes model.monitor, model.mon_mode = 'val_ZS_val_layer_val_acc', 'max' # Saving pointer for the layers. It is used for accessing layer activations # during debugging. model.output_layers_dict = dict(ZS_train_layer=ZS_train_layer, ZS_val_layer=ZS_val_layer, semantic_embed_layer=semantic_embed_layer) # Define the loss def LAGO_train_loss(y_true, y_pred): """ Wraps the LAGO_loss according to Keras API """ return LAGO_loss(y_true, y_pred, semantic_embed_layer.kernel, ZS_train_layer.class_descriptions.T, ZS_train_layer.Gamma, attributes_groups_ranges_ids) def zero_loss(y_true, y_pred): """ ZERO loss (above the validation classes head) according to Keras API. This makes sure that no computations are made through the validation classes head. """ return K.constant(0.) model.loss_list = [LAGO_train_loss, zero_loss, LAGO_attr_loss] return model
def proposed(type, train, test, code, epoch, batch): # Load MNIST train and test data X_train = np.loadtxt(train, delimiter=',', dtype=None) X_test = np.loadtxt(test, delimiter=',', dtype=None) # z_list : define experiment code(Z) size z_list = [code] autoencoder = [[] for i in range(len(z_list))] # E : epoch, BS = batch size E = epoch BS = batch # Train model and save data(code(Z), output and total loss data) model_index = 0 total_summary_loss_data = [ 'model_type', 'z_size', 'train_loss', 'test_loss' ] for z_size in z_list: # Define models INPUT_SIZE = 784 HIDDEN_SIZE1 = 2000 HIDDEN_SIZE2 = z_size if type == "digit": w_initializer = initializers.Orthogonal(gain=1.0, seed=None) b_initializer = initializers.random_normal(mean=0.0, stddev=0.05, seed=None) else: w_initializer = initializers.RandomUniform(minval=-0.05, maxval=0.05, seed=None) b_initializer = initializers.glorot_normal(seed=None) dense1 = Input(shape=(INPUT_SIZE, )) dense2 = Dense(HIDDEN_SIZE1, activation='relu', kernel_initializer=w_initializer, bias_initializer=b_initializer)(dense1) dense3 = Dense(HIDDEN_SIZE2, activation='linear', kernel_initializer=w_initializer, bias_initializer=b_initializer)(dense2) dense4 = Dense(HIDDEN_SIZE1, activation='relu', kernel_initializer=w_initializer, bias_initializer=b_initializer)(dense3) dense5 = Dense(INPUT_SIZE, activation='linear', kernel_initializer=w_initializer, bias_initializer=b_initializer)(dense4) autoencoder[model_index] = Model(dense1, dense5) sgd = optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) autoencoder[model_index].compile(loss='mean_squared_error', optimizer=sgd) autoencoder[model_index].fit(X_train, X_train, epochs=E, batch_size=BS, verbose=0) # Get output and calculate loss get_output = K.function([autoencoder[model_index].layers[0].input], [autoencoder[model_index].layers[4].output]) train_output = get_output([X_train])[0] test_output = get_output([X_test])[0] train_loss = np.sum((X_train - train_output)** 2) / (X_train.shape[0] * X_train.shape[1]) test_loss = np.sum( (X_test - test_output)**2) / (X_test.shape[0] * X_test.shape[1]) summary_loss_data = ['LAE', z_size, train_loss, test_loss] total_summary_loss_data = np.vstack( (total_summary_loss_data, summary_loss_data)) np.savetxt("total_loss.csv", total_summary_loss_data, delimiter=',', fmt='%s') # Get code(Z) get_z = K.function([autoencoder[model_index].layers[0].input], [autoencoder[model_index].layers[2].output]) test_z = get_z([X_test])[0] np.savetxt("test_code.csv", test_z, delimiter=',') model_index = model_index + 1 # Calculate and save LAE reconstruct (* LAE reconstruct = LAE output) for z_index in range(0, len(z_list)): z_size = z_list[z_index] test_recon = [[]] get_z = K.function([autoencoder[z_index].layers[0].input], [autoencoder[z_index].layers[2].output]) temp_Z = get_z([X_test])[0] for data_index in range(0, 10000): temp_vec = np.dot( temp_Z[data_index], autoencoder[z_index].layers[3].get_weights( )[0]) + autoencoder[z_index].layers[3].get_weights()[1] B = [] for k in range(0, len(temp_vec)): if (temp_vec[k] >= 0.0): B.extend([k]) w34 = autoencoder[z_index].layers[3].get_weights()[0] w34 = w34[:, B] w45 = autoencoder[z_index].layers[4].get_weights()[0] w45 = w45[B, :] W35 = np.dot(w34, w45) b3 = autoencoder[z_index].layers[3].get_weights()[1] b3 = b3[B] b4 = autoencoder[z_index].layers[4].get_weights()[1] temp_recon = np.dot(temp_Z[data_index], W35) + np.dot(b3, w45) + b4 if (data_index == 0): test_recon = temp_recon else: test_recon = np.vstack((test_recon, temp_recon)) if ((data_index + 1) % 1000 == 0): print(str(data_index + 1) + " finish!") np.savetxt("test_out.csv", test_recon, delimiter=',') # Print total loss print(total_summary_loss_data) print("learning proposed autoencoder modol finish! \n")
def get_lstm(n_units): return LSTM(units=n_units, return_sequences=True, kernel_initializer=initializers.glorot_uniform(seed=0), recurrent_initializer=initializers.Orthogonal(seed=0))
def build_model(input_shape: tuple, gpu, write_result_out_dir, pre_model=None, freeze=False, noise=None, verbose=True, savefig=True): if gpu: from keras.layers import CuDNNLSTM as LSTM else: from keras.layers import LSTM # construct the model input_layer = Input(input_shape) if noise: noise_input = GaussianNoise(np.sqrt(noise))(input_layer) dense = TimeDistributed( Dense(10, kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.glorot_uniform(seed=0), bias_initializer=initializers.Zeros()))(noise_input) else: dense = TimeDistributed( Dense(10, kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.glorot_uniform(seed=0), bias_initializer=initializers.Zeros()))(input_layer) lstm1 = LSTM(60, return_sequences=True, kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.glorot_uniform(seed=0), recurrent_initializer=initializers.Orthogonal(seed=0), bias_initializer=initializers.Zeros())(dense) lstm1 = BatchNormalization()(lstm1) lstm2 = LSTM(60, return_sequences=False, kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.glorot_uniform(seed=0), recurrent_initializer=initializers.Orthogonal(seed=0), bias_initializer=initializers.Zeros())(lstm1) lstm2 = BatchNormalization()(lstm2) output_layer = Dense( 1, activation='sigmoid', kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.glorot_uniform(seed=0), bias_initializer=initializers.Zeros())(lstm2) model = Model(inputs=input_layer, outputs=output_layer) if savefig: plot_model(model, to_file=f'{write_result_out_dir}/architecture.png', show_shapes=True, show_layer_names=False) # transfer weights from pre-trained model if pre_model: for i in range(2, len(model.layers) - 1): model.layers[i].set_weights(pre_model.layers[i].get_weights()) if freeze: model.layers[i].trainable = False model.compile(optimizer=Adam(), loss='mse', metrics=['accuracy']) if verbose: print(model.summary()) return model
initializers.random_uniform(minval=-0.2, seed=42), dict(class_name="random_uniform", minval=-0.2, maxval=0.05, seed=42), id="ru_1", ), pytest.param( initializers.TruncatedNormal(0.1), dict(class_name="truncated_normal", mean=0.1, stddev=0.05, seed=None), id="tn_0", ), pytest.param( initializers.truncated_normal(mean=0.2, stddev=0.003, seed=42), dict(class_name="truncated_normal", mean=0.2, stddev=0.003, seed=42), id="tn_1", ), pytest.param( initializers.Orthogonal(1.1), dict(class_name="orthogonal", gain=1.1, seed=None), id="o_0", ), pytest.param( initializers.orthogonal(gain=1.2, seed=42), dict(class_name="orthogonal", gain=1.2, seed=42), id="o_1", ), pytest.param(initializers.Identity(1.1), dict(class_name="identity", gain=1.1), id="i_0"), pytest.param(initializers.identity(), dict(class_name="identity", gain=1.0), id="i_1"), #################### VarianceScaling #################### pytest.param( initializers.glorot_normal(), dict(class_name="glorot_normal", seed=None), id="gn_0" ), pytest.param(
os.environ["CUDA_VISIBLE_DEVICES"] ='0,1' # 按需进行设置,如果只用一张显卡,可以忽略下面的multi_gpu_model(model, gpus=n)及其相关语句 if __name__ == '__main__': Xception_model = Xception(weights=None, include_top=False, input_shape=(299,299,3),pooling='Average') #实际上也可以设置weights='imagenet',不过这样做的话框架判断用户目录下,即~/.keras/models判断是否有对应的权重文件, # 没有的话会执行下载,建议自己另外加载就好。具体参看http://192.168.3.126/Kwong/totem_begin/blob/master/README.md # Xception_model = load_model('xception_weights_tf_dim_ordering_tf_kernels_notop.h5') # 如果加载的是imageNet预训练权重,需要添加另外的预处理方法(去imageNet数据集的均值和标准差) # 详见:http://192.168.3.126/Kwong/totem_begin/blob/master/tricks_in_processing_and_training/README.md top_model_avg = Sequential() top_model_avg.add(Flatten(input_shape=Xception_model.output_shape[1:],name='flatten_Average')) #top_model_avg.add(Dropout(0.5)) top_model_avg.add(BatchNormalization()) top_model_avg.add(Dense(48, name='dense_1', kernel_initializer=initializers.Orthogonal())) top_model_avg.add(advanced_activations.PReLU(alpha_initializer='zeros')) #top_model_avg.add(Dropout(0.5)) top_model_avg.add(BatchNormalization()) top_model_avg.add(Dense(2, name='dense_2', activation="softmax", kernel_initializer=initializers.Orthogonal())) model = Sequential() model.add(Xception_model) model.add(top_model_avg) # 以上是使用序列式定义模型的方法 #Xception_model2 = Xception(weights=None, include_top=False, input_shape=(299,299,3),pooling='Average') #base_out = Flatten()(Xception_model2.output) #drout1= Dropout(0.5)(base_out) #dense1= Dense(64,activation='relu',kernel_initializer=initializers.he_normal(seed=None))(drout1) #drout2= Dropout(0.5)(dense1)