def build(self, input_shape): self.kernel = self.add_weight(shape=(input_shape[1], 1), initializer='uniform', trainable=True, regularizer=l1(0.1), constraint=nonneg()) super(WeightLayer, self).build(input_shape)
def test_nonneg(self): from keras.constraints import nonneg nonneg_instance = nonneg() normed = nonneg_instance(K.variable(self.example_array)) assert (np.all(np.min(K.eval(normed), axis=1) == 0.))
def _build_model(self, data_dim): # Neural Net for Deep-Q learning Model minputs = Input(shape=(data_dim, )) # $ Arm a = Dense(64, activation='sigmoid')(minputs) a = Dense(64, activation='sigmoid')(a) a = Dense(32, activation='sigmoid')(a) a = Dense(32, activation='sigmoid')(a) a = Dense(1, activation='sigmoid', W_constraint=nonneg())(a) #reward comilation r = concatenate([a, minputs]) r = Dense(512, activation='relu')(r) r = Dense(128, activation='relu')(r) r = Dense(128, activation='relu')(r) r = Dense(64, activation='relu')(r) output = Dense(3, activation='linear')(r) modelReward = Model(inputs=minputs, outputs=output) modelPred = Model(inputs=modelReward.input, outputs=[modelReward.get_layer('dense_5').output]) plot_model(modelReward, to_file='modelDQNfull.png') plot_model(modelPred, to_file='modelDQNPred.png') plot_model(modelReward, to_file='dqnmodel.png', show_shapes=True, show_layer_names=False) print(modelReward.summary()) adadelta = optimizers.Adadelta() modelReward.compile(loss='mse', optimizer=adadelta, metrics=['accuracy']) modelPred.compile(loss='mse', optimizer=adadelta, metrics=['accuracy']) return modelReward, modelPred
def loadOrCreateModel(modelName, x): jsonName = "{}.json".format(modelName) h5Name = "{}.h5".format(modelName) if (isfile(jsonName) and isfile(h5Name)): loaded_model_json = None with open(jsonName, "r") as json_file: loaded_model_json = json_file.read() model = model_from_json(loaded_model_json) model.load_weights(h5Name) model.compile(loss="mse", optimizer="rmsprop", metrics=["mse"]) return model else: model = Sequential() model.add( LSTM(128, input_shape=(x.shape[1], x.shape[2]), return_sequences=True)) model.add(LSTM(64, activation="relu", return_sequences=True)) model.add(LSTM(32, activation="relu", return_sequences=False)) # model.add(Dense(128, activation="relu")) model.add(Dense(1, activation='linear', kernel_constraint=nonneg())) # model.add(Dense(1, activation="relu", kernel_constraint=nonneg())) model.compile(loss="mse", optimizer="rmsprop", metrics=["mse"]) model.summary() return model
def build(self, input_shape): if self.noneg == 0: self.alpha = self.add_weight( name='kernel', shape=(1, ), initializer=keras.initializers.Constant(value=self.ival), constraint=nonneg(), trainable=True) else: self.alpha = self.add_weight( name='kernel', shape=(1, ), initializer=keras.initializers.Constant(value=self.ival), constraint=nonneg(), trainable=False) super(Scale, self).build(input_shape)
def baseline_Model_2(): model = Sequential() # ~model.add(Dense(24, input_dim=24, kernel_initializer='normal', activation='relu')) # ~model.add(Dense(32, input_dim=32, kernel_initializer='normal', activation='relu')) model.add( Dense(34, input_dim=34, kernel_initializer='normal', activation='relu')) model.add(Dense(34, kernel_initializer='normal', activation='relu')) model.add(Dense(1, kernel_initializer='normal', W_constraint=nonneg())) model.compile(loss='mean_squared_error', optimizer='adam') return model
def build_model(input_size): regulizers = [L1L2(l1=0.0, l2=0.0)] model = Sequential() model.add(LSTM(32, input_shape=(look_back, input_size))) #model.add(Dropout(0.2)) #model.add(LSTM(10, input_shape=(8, look_back, input_size))) model.add(Dense(64, activation='relu')) #model.add(Dropout(0.2)) #model.add(Dense(n_seq)) model.add(Dense(n_seq, W_constraint=nonneg())) model.compile(loss='mean_squared_error', optimizer='adam') #, metrics=['mse', 'mae', rmse]) return model
def build(self, input_shape): self.alpha = self.add_weight( name='kernel', shape=(1, ), initializer=keras.initializers.Constant( self.ialpha), #constraint=WeightClip(0, 1.0), # 0<alpha<1 trainable=True) self.tau = self.add_weight( name='kernel', shape=(1, ), initializer=keras.initializers.Constant(self.itau), constraint=nonneg(), # Positive threshold trainable=True) super(Prox, self).build(input_shape)
def build_socnn(input_shape_sig=(128, 1), input_shape_off=(128, 1), dim=1): #significant_network Input_sig = Input(shape=input_shape_sig, dtype='float32', name='input_sig') name = "Significance_Conv_0" x = Conv1D(filters=8,kernel_size=ks, padding='same', activation='linear', name=name, kernel_constraint=maxnorm(norm))(Input_sig) for i in range(num_layer_sig-1): name = "Significance_Conv_" + str(i+1) if i == (num_layer_sig-2): fn = dim-1 else: fn = 8 x = Conv1D(filters=fn, kernel_size=ks, padding='same', activation='linear', name=name, kernel_constraint=maxnorm(norm))(x) x = BatchNormalization(name="Significance_BN"+str(i+1))(x) output_sig = x #offset_network Input_off = Input(shape=input_shape_off, dtype='float32', name='input_off') name = "Offset_Conv_0" y = Conv1D(filters=dim-1, kernel_size=ks, padding='same', activation='linear', name=name, kernel_constraint=maxnorm(norm))(Input_off) output_off = keras.layers.add([y, Input_off], name='output_off') value = Permute((2, 1))(output_off) output_sig = Permute((2, 1))(output_sig) output_sig = TimeDistributed(Activation('softmax'), name='softmax')(output_sig) #Hn-1 = 𝝈(𝑺) ⨂(𝐨𝐟𝐟+𝒙𝑰) H1 = keras.layers.multiply(inputs=[output_sig, value], name='significancemerge') #Hn H2 = TimeDistributed(Dense(output_length, activation='linear', use_bias=False, kernel_constraint=nonneg() if nonnegative else None), name='out')(H1) main_output = Permute((2, 1), name='main_output')(H2) model = keras.models.Model(inputs=[Input_sig, Input_off], outputs=[main_output, output_off]) model.compile(optimizer=keras.optimizers.Adam(lr=lr, clipnorm=clipnorm), loss={'main_output': 'mse', 'output_off': 'mse'}, loss_weights={'main_output': 1., 'output_off': aux_weight}) return model
def get_flow_model(self, P_init): s = kl.Input((self.num_states, )) Dense_W = kl.Dense(self.num_states, use_bias=False, W_constraint=nonneg(), weights=[np.array(P_init)]) logit = Dense_W(s) #P = kl.Activation('softmax')(logit) P = kl.Lambda(lambda z: z / K.sum(z, axis=-1, keepdims=True))(logit) model = km.Model(inputs=[s], outputs=P) #model.summary() model.compile(loss='categorical_crossentropy', optimizer='adam') return model, Dense_W
def baseline_Model(): model = Sequential() model.add( Dense(24, input_dim=24, kernel_initializer='normal', activation='relu')) # ~model.add(Dense(25, input_dim=25, kernel_initializer='normal', activation='relu')) # ~model.add(Dense(32, input_dim=32, kernel_initializer='normal', activation='relu')) # ~model.add(Dense(34, input_dim=34, kernel_initializer='normal', activation='relu')) model.add(Dense(6, kernel_initializer='normal', activation='relu')) model.add( Dense(1, kernel_initializer='normal', activation='linear', W_constraint=nonneg()) ) #Does not work if input is transformed onto [-1,1] # ~model.add(Dense(1, kernel_initializer='normal')) model.compile(loss='mean_squared_error', optimizer='adam') return model
def Dense_Model(params,inputs,lr=1e-4): import keras from keras.models import Sequential from keras.layers import Dense, Activation, Dropout from keras.wrappers.scikit_learn import KerasRegressor from keras.callbacks import EarlyStopping,ModelCheckpoint,LearningRateScheduler import tensorflow as tf from keras.constraints import nonneg # patience=10 config = tf.compat.v1.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = params['Memory'] session = tf.compat.v1.Session(config=config) model = Sequential()#'relu' NUM_GPU = 1 # or the number of GPUs available on your machin adam = keras.optimizers.Adam(lr = lr) gpu_list = [] initializer = keras.initializers.glorot_uniform(seed=params['iteration']) # print(params['Save']['Weights']) for i in range(NUM_GPU): gpu_list.append('gpu(%d)' % i) if params['Loss'] == 'Boot_Loss': model.add(Dense(params['N'], input_dim=inputs,activation='relu',kernel_initializer=initializer,kernel_constraint=nonneg())) model.add(Dense(1,activation='elu',kernel_constraint=nonneg())) model.compile(loss=Boot_Loss, optimizer='adam') elif params['HiddenLayers']==1: model.add(Dense(params['N'], input_dim=inputs,activation=params['Activation'],kernel_initializer=initializer)) # model.add(Dropout(0.1)) # model.add(Dense(params['N'], input_dim=inputs,activation='sigmoid',kernel_initializer=initializer))#,kernel_constr model.add(Dense(1)) # model.add(Dense(1,activation='elu',kernel_constraint=nonneg())) model.compile(loss=params['Loss'], optimizer='adam') else: model.add(Dense(params['N'], input_dim=inputs,activation=params['Activation'],kernel_initializer=initializer)) model.add(Dropout(0.1)) model.add(Dense(int(params['N']/2), activation=params['Activation'])) model.add(Dense(1)) model.compile(loss=params['Loss'], optimizer='adam')#,context=gpu_list) # - Add if using MXNET if params['Save']['Weights'] == True: # callbacks = [EarlyStopping(monitor='val_loss', patience=params['patience'],verbose=0),#params['Verbose']), # ModelCheckpoint(filepath=params['Spath']+params['Sname']+str(params['iteration'])+'.h5', monitor='val_loss', save_best_only=True)] callbacks = [EarlyStopping(monitor='val_loss', patience=params['patience'],verbose=0),#params['Verbose']), ModelCheckpoint(filepath=params['Spath']+params['Sname']+str(params['iteration'])+'.h5', monitor='loss', save_best_only=True)] else: callbacks = [EarlyStopping(monitor='val_loss', patience=params['patience'])] return(model,callbacks)
def build_model(): model = Sequential() model.add(Dense(500, input_dim=122, activation='relu')) model.add(Dropout(0.3)) model.add(Dense(200, activation='relu')) model.add(Dropout(0.3)) model.add(Dense(100, activation='sigmoid')) model.add(Dropout(0.3)) model.add(Dense(1, W_constraint=nonneg(), activation='sigmoid')) optimizers = [] optimizers.append(SGD(lr=.1, momentum=0.1, decay=0.0)) optimizers.append(RMSprop(lr=0.001, rho=0.9, epsilon=1e-06)) optimizers.append(Adagrad(lr=0.01, epsilon=1e-06)) optimizers.append(Adadelta(lr=1.0, rho=0.95, epsilon=1e-06)) optimizers.append( Adam(lr=0.0001 / 2, beta_1=0.9, beta_2=0.999, epsilon=1e-08)) optimizers.append(Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08)) model.compile(loss='mean_squared_error', optimizer=optimizers[4]) return model
def get_bs_particle_graphs(self, s, Dense_W, V, R, temp): Dense_W(s) W = Dense_W.weights[0] Dense_W_ = kl.Dense(self.num_states, use_bias=False, W_constraint=nonneg()) Dense_W_(s) W_ = Dense_W_.weights[0] s_embed = kl.Dense(self.num_states, activation='tanh')(s) Dense_E = kl.Dense(self.num_states * self.num_states) null_input = kl.Lambda(lambda z: 0 * z[:, 0:1])(s_embed) E_logit = kl.Dense(self.num_states, activation='tanh')(null_input) E_logit = Dense_E(E_logit) E_logit = kl.Lambda(lambda z: z / temp)(E_logit) E = kl.Activation('sigmoid')(E_logit) E = kl.Reshape((self.num_states, self.num_states))(E) E = kl.Lambda(lambda z: self.mask * z)(E) Dense_W.trainable = False logit_1 = Dense_W(s) logit_2 = kl.Lambda(lambda z: W_ * (z))(E) logit_2 = kl.Dot(axes=1)([logit_2, s]) logit = kl.Add()([logit_1, logit_2]) #P = kl.Activation('softmax')(logit) P = kl.Lambda(lambda z: z / K.sum(z, axis=-1, keepdims=True))(logit) V_ = kl.Dot(axes=-1)([P, V]) r = kl.Dot(axes=-1)([s, R]) v = kl.Lambda(lambda z: z[0] + self.gamma * z[1])([r, V_]) return W, W_, E, P, V_, v
encoder_input = Input(shape = X_train.shape[1:],name = 'encoder_input') decoder_input = Input(shape = (None,latent_dim),name = 'decoder_input') permute_conv = Permute((2,3,1)) conv1 = TimeDistributed(Conv1D(4, kernel_size = 1),input_shape = (X_train.shape[1],X_train.shape[2],X_train.shape[3])) conv2 = TimeDistributed(Conv1D(2, kernel_size = 1),input_shape = (X_train.shape[1],X_train.shape[2],X_train.shape[3])) conv3 = TimeDistributed(Conv1D(1, kernel_size = 1),input_shape = (X_train.shape[1],X_train.shape[2],X_train.shape[3])) concat = Concatenate() dropout = Dropout(dropout_rate) permute = Permute((2,1,3)) reshape = Reshape((look_back_period,X_train.shape[2])) encoderLSTM = LSTM(units = latent_dim,return_state = True,return_sequences = True,name = 'enc_LSTM',dropout = dropout_rate) encoderLSTM2 = LSTM(units = latent_dim,return_state = True,return_sequences = True,name = 'enc_LSTM2',dropout = dropout_rate) decoderLSTM = LSTM(units = latent_dim,return_state = True,return_sequences = True,name = 'dec_LSTM',dropout = dropout_rate) reshape2 = Reshape((look_back_period,X_train.shape[2])) dense_output = TimeDistributed(Dense(X_train.shape[2], activation = 'relu', W_constraint=nonneg()),name = 'time_distirbuted_dense_output') ## ## ## encoder conv1_out = conv1(encoder_input) conv2_out = conv2(conv1_out) conv3_out = conv3(conv2_out) reshape_out_1 = reshape(conv3_out) encoder_out = encoderLSTM(reshape_out_1) #encoder_out = encoderLSTM2(encoder_out[0]) encoder_states = encoder_out[1:] encoder_out = encoder_out[0]
def test_nonneg(): nonneg_instance = constraints.nonneg() normed = nonneg_instance(K.variable(example_array)) assert(np.all(np.min(K.eval(normed), axis=1) == 0.))
model.add(Convolution2D(40, 3, 3)) model.add(LeakyReLU()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) ''' model.add(Flatten()) model.add(Dense(256)) model.add(LeakyReLU()) ''' model.add(Dense(256)) model.add(LeakyReLU()) model.add(Dropout(0.50)) ''' model.add(Dense(1, W_constraint=nonneg())) model.add(Activation('linear')) model.compile(loss=rmse, optimizer='adam') print("--- Compiling model: %s minutes ---" % round(((time.time() - start_time)/60),2)) start_time = time.time() # Try to work around class inbalance, increases the training size a LOT ''' df_train2 = pd.DataFrame() df_train2 = df_train2.append(df_train.ix[np.repeat(df_train.index[(y_train >= 1.0) & (y_train < 1.5)].tolist(), 7)]) df_train2 = df_train2.append(df_train.ix[np.repeat(df_train.index[(y_train >= 1.5) & (y_train < 2.0)].tolist(), 5)]) df_train2 = df_train2.append(df_train.ix[np.repeat(df_train.index[(y_train >= 2.0) & (y_train < 2.5)].tolist(), 1)]) df_train2 = df_train2.append(df_train.ix[np.repeat(df_train.index[(y_train >= 2.5) & (y_train <= 3)].tolist(), 1)]) df_train = df_train2
def get_pnet(inputs, features, genes, n_hidden_layers, direction, activation, activation_decision, w_reg, w_reg_outcomes, dropout, sparse, add_unk_genes, batch_normal, kernel_initializer, use_bias=False, shuffle_genes=False, attention=False, dropout_testing=False, non_neg=False, sparse_first_layer=True): feature_names = {} n_features = len(features) n_genes = len(genes) if not type(w_reg) == list: w_reg = [w_reg] * 10 if not type(w_reg_outcomes) == list: w_reg_outcomes = [w_reg_outcomes] * 10 if not type(dropout) == list: dropout = [w_reg_outcomes] * 10 w_reg0 = w_reg[0] w_reg_outcome0 = w_reg_outcomes[0] w_reg_outcome1 = w_reg_outcomes[1] reg_l = l2 constraints = {} if non_neg: from keras.constraints import nonneg constraints = {'kernel_constraint': nonneg()} # constraints= {'kernel_constraint': nonneg(), 'bias_constraint':nonneg() } if sparse: if shuffle_genes == 'all': ones_ratio = float(n_features) / np.prod([n_genes, n_features]) logging.info('ones_ratio random {}'.format(ones_ratio)) mapp = np.random.choice([0, 1], size=[n_features, n_genes], p=[1 - ones_ratio, ones_ratio]) layer1 = SparseTF(n_genes, mapp, activation=activation, W_regularizer=reg_l(w_reg0), name='h{}'.format(0), kernel_initializer=kernel_initializer, use_bias=use_bias, **constraints) # layer1 = Diagonal(n_genes, input_shape=(n_features,), activation=activation, W_regularizer=l2(w_reg), # use_bias=use_bias, name='h0', kernel_initializer= kernel_initializer ) else: layer1 = Diagonal(n_genes, input_shape=(n_features, ), activation=activation, W_regularizer=l2(w_reg0), use_bias=use_bias, name='h0', kernel_initializer=kernel_initializer, **constraints) else: if sparse_first_layer: # layer1 = Diagonal(n_genes, input_shape=(n_features, ), activation=activation, W_regularizer=l2(w_reg0), use_bias=use_bias, name='h0', kernel_initializer=kernel_initializer, **constraints) else: layer1 = Dense(n_genes, input_shape=(n_features, ), activation=activation, W_regularizer=l2(w_reg0), use_bias=use_bias, name='h0', kernel_initializer=kernel_initializer) outcome = layer1(inputs) if attention: attention_probs = Diagonal(n_genes, input_shape=(n_features, ), activation='sigmoid', W_regularizer=l2(w_reg0), name='attention0')(inputs) outcome = multiply([outcome, attention_probs], name='attention_mul') decision_outcomes = [] # if reg_outcomes: # decision_outcome = Dense(1, activation='linear', name='o_linear{}'.format(0), W_regularizer=reg_l(w_reg_outcome0), **constraints)(inputs) decision_outcome = Dense(1, activation='linear', name='o_linear{}'.format(0), W_regularizer=reg_l(w_reg_outcome0))(inputs) # else: # decision_outcome = Dense(1, activation='linear', name='o_linear{}'.format(0))(inputs) # testing if batch_normal: decision_outcome = BatchNormalization()(decision_outcome) # decision_outcome = Activation( activation=activation_decision, name='o{}'.format(0))(decision_outcome) # first outcome layer # decision_outcomes.append(decision_outcome) # if reg_outcomes: # decision_outcome = Dense(1, activation='linear', name='o_linear{}'.format(1), W_regularizer=reg_l(w_reg_outcome1/2.), **constraints)(outcome) decision_outcome = Dense(1, activation='linear', name='o_linear{}'.format(1), W_regularizer=reg_l(w_reg_outcome1 / 2.))(outcome) # else: # decision_outcome = Dense(1, activation='linear', name='o_linear{}'.format(1))(outcome) # drop2 = Dropout(dropout, name='dropout_{}'.format(0)) drop2 = Dropout(dropout[0], name='dropout_{}'.format(0)) outcome = drop2(outcome, training=dropout_testing) # testing if batch_normal: decision_outcome = BatchNormalization()(decision_outcome) decision_outcome = Activation(activation=activation_decision, name='o{}'.format(1))(decision_outcome) decision_outcomes.append(decision_outcome) if n_hidden_layers > 0: maps = get_layer_maps(genes, n_hidden_layers, direction, add_unk_genes) layer_inds = range(1, len(maps)) # if adaptive_reg: # w_regs = [float(w_reg)/float(i) for i in layer_inds] # else: # w_regs = [w_reg] * len(maps) # if adaptive_dropout: # dropouts = [float(dropout)/float(i) for i in layer_inds] # else: # dropouts = [dropout]*len(maps) print 'original dropout', dropout print 'dropout', layer_inds, dropout, w_reg w_regs = w_reg[1:] w_reg_outcomes = w_reg_outcomes[1:] dropouts = dropout[1:] for i, mapp in enumerate(maps[0:-1]): w_reg = w_regs[i] w_reg_outcome = w_reg_outcomes[i] # dropout2 = dropouts[i] dropout = dropouts[1] names = mapp.index # names = list(mapp.index) mapp = mapp.values if shuffle_genes in ['all', 'pathways']: mapp = shuffle_genes_map(mapp) n_genes, n_pathways = mapp.shape logging.info('n_genes, n_pathways {} {} '.format( n_genes, n_pathways)) # print 'map # ones {}'.format(np.sum(mapp)) print 'layer {}, dropout {} w_reg {}'.format(i, dropout, w_reg) layer_name = 'h{}'.format(i + 1) if sparse: hidden_layer = SparseTF(n_pathways, mapp, activation=activation, W_regularizer=reg_l(w_reg), name=layer_name, kernel_initializer=kernel_initializer, use_bias=use_bias, **constraints) else: hidden_layer = Dense(n_pathways, activation=activation, W_regularizer=reg_l(w_reg), name=layer_name, kernel_initializer=kernel_initializer, **constraints) outcome = hidden_layer(outcome) if attention: attention_probs = Dense(n_pathways, activation='sigmoid', name='attention{}'.format(i + 1), W_regularizer=l2(w_reg))(outcome) outcome = multiply([outcome, attention_probs], name='attention_mul{}'.format(i + 1)) # if reg_outcomes: # decision_outcome = Dense(1, activation='linear', name='o_linear{}'.format(i + 2), W_regularizer=reg_l( w_reg2/(2**i)))(outcome) # decision_outcome = Dense(1, activation='linear', name='o_linear{}'.format(i + 2), W_regularizer=reg_l( w_reg_outcome), **constraints)(outcome) decision_outcome = Dense( 1, activation='linear', name='o_linear{}'.format(i + 2), W_regularizer=reg_l(w_reg_outcome))(outcome) # else: # decision_outcome = Dense(1, activation='linear', name='o_linear{}'.format(i + 2))(outcome) # testing if batch_normal: decision_outcome = BatchNormalization()(decision_outcome) decision_outcome = Activation( activation=activation_decision, name='o{}'.format(i + 2))(decision_outcome) decision_outcomes.append(decision_outcome) drop2 = Dropout(dropout, name='dropout_{}'.format(i + 1)) outcome = drop2(outcome, training=dropout_testing) feature_names['h{}'.format(i)] = names # feature_names.append(names) i = len(maps) feature_names['h{}'.format(i - 1)] = maps[-1].index # feature_names.append(maps[-1].index) return outcome, decision_outcomes, feature_names
X_train = X_train.reshape(60000, 784) X_test = X_test.reshape(10000, 784) X_train = X_train.astype("float32") X_test = X_test.astype("float32") X_train /= 255 X_test /= 255 # convert class vectors to binary class matrices Y_train = np_utils.to_categorical(y_train, nb_classes) Y_test = np_utils.to_categorical(y_test, nb_classes) model = Sequential() model.add(Dense(784, 20, W_constraint=maxnorm(1))) model.add(Activation('relu')) model.add(Dropout(0.1)) model.add(Dense(20, 20, W_constraint=nonneg())) model.add(Activation('relu')) model.add(Dropout(0.1)) model.add(Dense(20, 10, W_constraint=maxnorm(1))) model.add(Activation('softmax')) rms = RMSprop() model.compile(loss='categorical_crossentropy', optimizer=rms) model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=0)
labels=[] for i in tmp: k=[0]*labelLen k[i]=1 labels.append(k) X_train=data Y_train=labels weights=readEmbedding() leftmodel = Sequential(name='Embedding') leftmodel.add(Embedding(input_length=maxlen,input_dim=vocabSize, output_dim=50, mask_zero= True, name='embedding',weights=weights, trainable=False)) rightmodel=Sequential(name='Importance') rightmodel.add(Embedding(input_length=maxlen,input_dim=vocabSize, output_dim=1, mask_zero= True, name='importance', trainable=True, W_constraint=nonneg())) model=Sequential() model.add(Merge([leftmodel,rightmodel], mode=(lambda x: x[0]*K.repeat_elements(x[1],50,2)) , output_shape=(maxlen,50), name='merge')) model.add(LSTM(output_dim=100, input_length=maxlen, activation='tanh', inner_activation='hard_sigmoid',return_sequences=False,name='lstm')) model.add(Dense(labels[0].__len__())) model.add(Activation('sigmoid')) model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['categorical_accuracy']) early_stopping = EarlyStopping(monitor='val_loss', patience=0.001) model.fit(x=[X_train,X_train], y=Y_train, batch_size=10, nb_epoch=50)
ReLUThres = 0.01 TNS.add(ThresholdedReLU(theta = ReLUThres, input_shape=(nDict,))) TNS.add(Dense(nDict, activation='linear', bias = True)) Z = TNS(W) nLayers = 8 for l in range(nLayers-1): Y = merge([Z,W],"sum") Z = TNS(Y) Y = merge([Z,W],"sum") T = ThresholdedReLU(theta = ReLUThres)(Y) Viso = Lambda(split_last, output_shape=split_last_output_shape)(T) Vother = Lambda(split_others, output_shape=split_others_output_shape)(T) normVother = Lambda(lambda x: (x+tau)/(x+tau).norm(1, axis = 1).reshape((x.shape[0],1)))(Vother) VicK = Dense(2, W_constraint = nonneg(), activation='linear', bias = True)(normVother) Kappa = Lambda(split_last, output_shape=split_last_output_shape)(VicK) Vic = Lambda(split_others, output_shape=split_others_output_shape)(VicK) OD = Lambda(lambda x: 2.0/np.pi*np.arctan(1.0/(x+tau)))(Kappa) weight = [1.0,1.0,1.0] epoch = 10 print "nLayers, ReLUThres, epoch, weight, nDict: ", nLayers, ReLUThres, epoch, weight, nDict ### fitting the model ### print "Fitting" clf = Model(input=inputs,output=[Vic,OD,Viso]) clf.compile(optimizer=Adam(lr=0.0001), loss='mse', loss_weights = weight)
TNS.add(ThresholdedReLU(theta=ReLUThres, input_shape=(nDict, ))) TNS.add(Dense(nDict, activation='linear', bias=True)) Z = TNS(W) nLayers = 8 for l in range(nLayers - 1): Y = merge([Z, W], "sum") Z = TNS(Y) Y = merge([Z, W], "sum") T = ThresholdedReLU(theta=ReLUThres)(Y) Viso = Lambda(split_last, output_shape=split_last_output_shape)(T) Vother = Lambda(split_others, output_shape=split_others_output_shape)(T) normVother = Lambda(lambda x: (x + tau) / (x + tau).norm(1, axis=1).reshape( (x.shape[0], 1)))(Vother) VicK = Dense(2, W_constraint=nonneg(), activation='linear', bias=True)(normVother) Kappa = Lambda(split_last, output_shape=split_last_output_shape)(VicK) Vic = Lambda(split_others, output_shape=split_others_output_shape)(VicK) OD = Lambda(lambda x: 2.0 / np.pi * np.arctan(1.0 / (x + tau)))(Kappa) weight = [1.0, 1.0, 1.0] epoch = 10 #epoch = 15 print("nLayers, ReLUThres, epoch, weight, nDict: %d %r %d %r %d" % (nLayers, ReLUThres, epoch, weight, nDict)) ### fitting the model ### print("Fitting")
def get_model(args, input_dim, output_dim): """Build neural network layers If using pretrained autoencoder, assumes the first three hidden layers match Args: args: dictionary containing experiment parameters, see get_args() input_dim: int size of input vector output_dim: int size of output vector Returns: Keras Model """ kc = None bc = None opt = get_opt(args['opt']) d0 = Dropout(args['drop'], seed=args['h1']) d1 = Dropout(args['drop'], seed=args['h2']) d2 = Dropout(args['drop'], seed=args['h3']) d3 = Dropout(args['drop'], seed=args['h4']) d4 = Dropout(args['drop'], seed=args['h5']) d5 = Dropout(args['drop'], seed=args['h5']) # build network input_genes = Input(shape=(input_dim, 1)) # add 3rd dimension for keras-vis e = Flatten()(input_genes) e = d0(e) e = Dense(args['h1'])(e) e = BatchNormalization()(e) e = Activation(args['act'])(e) e = d1(e) if args['h2'] > 0: e = Dense(args['h2'])(e) e = BatchNormalization()(e) e = Activation(args['act'])(e) e = d2(e) if args['h3'] > 0: e = Dense(args['h3'])(e) e = BatchNormalization()(e) e = Activation(args['act'])(e) e = d3(e) if args['h4'] > 0: e = Dense(args['h4'])(e) e = BatchNormalization()(e) e = Activation(args['act'])(e) e = d4(e) if args['h5'] > 0: e = Dense(args['h5'])(e) e = BatchNormalization()(e) e = Activation(args['act'])(e) e = d5(e) if args['predType'] == 'regression': metrics = ['mae', 'mape', r2] act = 'linear' if args['nonneg']: kc = nonneg() bc = nonneg() else: metrics = ['acc'] if args['predType'] == 'multiClass': act = 'softmax' else: act = 'sigmoid' # binary class predictor = Dense(output_dim, name='preds', kernel_constraint=kc, bias_constraint=bc)(e) predictor = Activation(act)(predictor) model = Model(input_genes, predictor) model.compile(loss=args['loss'], optimizer=opt, metrics=metrics) model.summary() if args['pretrain'] is not None: print(' set up ae') # set pre-trained weights from ae ae = load_model(args['ae_model'], custom_objects={'r2': r2}) # just set the weights model.layers[3].set_weights(ae.layers[1].get_weights()) # 3 dropout layers model.layers[7].set_weights(ae.layers[4].get_weights()) model.layers[11].set_weights(ae.layers[7].get_weights()) if args['freeze'] == 1: # freeze layers print(' freeze weights') if args['num_ae_layers'] > 0: model.layers[3].trainable = False # 3 dropout layers if args['num_ae_layers'] > 1: model.layers[7].trainable = False del ae model = Model(input_genes, predictor) model.compile(loss=args['loss'], optimizer=opt, metrics=metrics) return model
loss=dat-(tf.matmul(label,mu)) loss=tf.trace(tf.matmul(tf.scalar_mul(1.0/sampleSize,loss),loss,transpose_b=True)) norm = tf.reduce_sum(label, 1, keep_dims=True) norm = tf.scalar_mul( 100.0, tf.add(norm, minus1)) norm = tf.matmul(norm,norm,transpose_a=True) loss= tf.add(loss,norm) loss=tf.reshape(loss,(1,1)) return loss opt=RMSprop(lr=0.01, rho=0.5, epsilon=1e-08, decay=0.0) #cluster network labelID=Input(batch_shape=(1,sampleSize)) labelEmbedding=Embedding(input_dim=sampleSize, input_length=sampleSize, output_dim=alpha,W_constraint=kc.nonneg() ,name='embedding')(labelID) rawInput=Input(batch_shape=(1,sampleSize*dim)) minus1Input=Input(batch_shape=(1,sampleSize)) #rawInput=Dense(input_dim=sampleSize*dim , output_dim=sampleSize* dim, weights=np.eye(sampleSize*dim),activation='linear', trainable=False)(rawInput) out=merge(inputs=[rawInput,labelEmbedding,minus1Input], mode= LossFunction, output_shape=(1,1)) model=Model(input=[rawInput,labelID,minus1Input],output=out) model.compile(optimizer=opt, loss='mse', metrics=['mse']) X=X_train.reshape((1,sampleSize*dim)) X2=np.asanyarray(range(X_train.__len__())).reshape(1,sampleSize) X3=np.asanyarray([-1.0]*sampleSize).reshape(1,sampleSize) target=np.asanyarray([[[0.0]]])
x.append(i) pyplot.plot(x, inv_testY, x, inv_yhat) pyplot.show() else: #Set network model = Sequential() model.add( LSTM(50, input_shape=(train_X.shape[1], train_X.shape[2]), return_sequences=True)) model.add(Dropout(0.25)) model.add(LSTM(20)) model.add(Dense(1, activation='linear', kernel_constraint=nonneg())) model.compile(loss="mse", optimizer="Nadam", metrics=["mse"]) history = model.fit(train_X, train_y, epochs=40, batch_size=n_train_weeks, validation_data=(test_X, test_y), verbose=2, shuffle=False) #Save model model_json = model.to_json() with open("model.json", "w") as json_file: json_file.write(model_json) #seralize weights to HDF5