def build(i): #input = Input(shape=(sentence_length,)) # embedding embedding = Embedding(len(vocab), WORD_EMBEDDING_DIM) input_embedding = embedding(i) conv = Convolution1D( nb_filter=NB_FILTER, filter_length=FILTER_LENGTH, border_mode='same', #activation='tanh', subsample_length=1, W_constraint = maxnorm(3), b_constraint = maxnorm(3), #input_shape=(AMAX_TIMESTAMP, WORD_EMBEDDING_DIM), #name='conv' ) # dropout = Dropout(0.5) # dropout input_dropout = conv(input_embedding) # maxpooling maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) input_pool = maxpool(input_dropout) # activation activation = Activation('tanh') output = activation(input_pool) return output
def buildConvolution(self, name): filters = self.params.get('filters') nb_filter = self.params.get('nb_filter') assert filters assert nb_filter convs = [] for fsz in filters: layer_name = '%s-conv-%d' % (name, fsz) conv = Convolution1D( nb_filter=nb_filter, filter_length=fsz, border_mode='valid', #activation='relu', subsample_length=1, init='glorot_uniform', #init=init, #init=lambda shape, name: initializations.uniform(shape, scale=0.01, name=name), W_constraint=maxnorm(self.params.get('w_maxnorm')), b_constraint=maxnorm(self.params.get('b_maxnorm')), #W_regularizer=regularizers.l2(self.params.get('w_l2')), #b_regularizer=regularizers.l2(self.params.get('b_l2')), #input_shape=(self.q_length, self.wdim), name=layer_name ) convs.append(conv) self.layers['%s-convolution' % name] = convs
def build_embedding_layer(args): try: n_embeddings = args.n_vocab except AttributeError: n_embeddings = args.n_embeddings try: mask_zero = args.mask_zero except AttributeError: mask_zero = False if hasattr(args, 'embedding_weights') and args.embedding_weights is not None: W = np.load(args.embedding_weights) if args.train_embeddings is True or args.train_embeddings == 'true': return Embedding(n_embeddings, args.n_embed_dims, weights=[W], input_length=args.input_width, W_constraint=maxnorm(args.embedding_max_norm), mask_zero=mask_zero) else: return ImmutableEmbedding(n_embeddings, args.n_embed_dims, weights=[W], mask_zero=mask_zero, input_length=args.input_width) else: if args.train_embeddings is True: return Embedding(n_embeddings, args.n_embed_dims, init=args.embedding_init, W_constraint=maxnorm(args.embedding_max_norm), mask_zero=mask_zero, input_length=args.input_width) else: return ImmutableEmbedding(n_embeddings, args.n_embed_dims, init=args.embedding_init, mask_zero=mask_zero, input_length=args.input_width)
def create_model(): # create model model = Sequential() model.add(Dropout(0.2, input_shape=(60,))) model.add(Dense(60, init='normal', activation='relu', W_constraint=maxnorm(3))) model.add(Dense(30, init='normal', activation='relu', W_constraint=maxnorm(3))) model.add(Dense(1, init='normal', activation='sigmoid')) # Compile model sgd = SGD(lr=0.1, momentum=0.9, decay=0.0, nesterov=False) model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy']) return model
def setup_model_3b(patch_size, in_chan): inputs = Input((patch_size, patch_size, in_chan)) drop1 = Dropout(0.1)(inputs) conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(drop1) conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(conv1) pool1 = MaxPooling2D(pool_size=(2, 2))(conv1) drop2 = Dropout(0.05)(pool1) conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(drop2) conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(conv2) pool2 = MaxPooling2D(pool_size=(2, 2))(conv2) drop3 = Dropout(0.05)(pool2) conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(drop3) conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(conv3) pool3 = MaxPooling2D(pool_size=(2, 2))(conv3) flat = Flatten()(pool3) dense8 = Dense(64, activation='relu', kernel_constraint=maxnorm(3))(flat) dense9 = Dense(16, activation='relu', kernel_constraint=maxnorm(3))(dense8) output = Dense(1,activation='sigmoid')(dense9) model = Model(inputs=inputs, outputs=output) model.compile(optimizer=Adam(decay=0.0001), loss='binary_crossentropy', metrics=['accuracy', libs.metrics.f1_score, libs.metrics.precision, libs.metrics.recall]) print model.summary() return model
def test_maxnorm(): for m in test_values: norm_instance = constraints.maxnorm(m) normed = norm_instance(K.variable(example_array)) assert(np.all(K.eval(normed) < m)) # a more explicit example norm_instance = constraints.maxnorm(2.0) x = np.array([[0, 0, 0], [1.0, 0, 0], [3, 0, 0], [3, 3, 3]]).T x_normed_target = np.array([[0, 0, 0], [1.0, 0, 0], [2.0, 0, 0], [2. / np.sqrt(3), 2. / np.sqrt(3), 2. / np.sqrt(3)]]).T x_normed_actual = K.eval(norm_instance(K.variable(x))) assert_allclose(x_normed_actual, x_normed_target, rtol=1e-05)
def test_maxnorm(self): from keras.constraints import maxnorm for m in self.some_values: norm_instance = maxnorm(m) normed = norm_instance(self.example_array) assert (np.all(normed.eval() < m)) # a more explicit example norm_instance = maxnorm(2.0) x = np.array([[0, 0, 0], [1.0, 0, 0], [3, 0, 0], [3, 3, 3]]).T x_normed_target = np.array([[0, 0, 0], [1.0, 0, 0], [2.0, 0, 0], [2./np.sqrt(3), 2./np.sqrt(3), 2./np.sqrt(3)]]).T x_normed_actual = norm_instance(x).eval() assert_allclose(x_normed_actual, x_normed_target)
def build_model(): main_input = Input(shape=(maxlen, ), dtype='int32', name='main_input') embedding = Embedding(max_features, embedding_dims, weights=[np.matrix(W)], input_length=maxlen, name='embedding')(main_input) embedding = Dropout(0.50)(embedding) conv4 = Conv1D(filters=nb_filter, kernel_size=4, padding='valid', activation='relu', strides=1, name='conv4')(embedding) maxConv4 = MaxPooling1D(pool_size=2, name='maxConv4')(conv4) conv5 = Conv1D(filters=nb_filter, kernel_size=5, padding='valid', activation='relu', strides=1, name='conv5')(embedding) maxConv5 = MaxPooling1D(pool_size=2, name='maxConv5')(conv5) # x = merge([maxConv4, maxConv5], mode='concat') x = keras.layers.concatenate([maxConv4, maxConv5]) x = Dropout(0.15)(x) x = RNN(rnn_output_size)(x) x = Dense(hidden_dims, activation='relu', kernel_initializer='he_normal', kernel_constraint = maxnorm(3), bias_constraint=maxnorm(3), name='mlp')(x) x = Dropout(0.10, name='drop')(x) output = Dense(1, kernel_initializer='he_normal', activation='sigmoid', name='output')(x) model = Model(inputs=main_input, outputs=output) model.compile(loss='binary_crossentropy', # optimizer=Adadelta(lr=0.95, epsilon=1e-06), # optimizer=Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=0.0), # optimizer=Adagrad(lr=0.01, epsilon=1e-08, decay=1e-4), metrics=["accuracy"]) return model
def build_model(): print('Build model...%d of %d' % (i + 1, folds)) main_input = Input(shape=(maxlen, ), dtype='int32', name='main_input') embedding = Embedding(max_features, embedding_dims, weights=[np.matrix(W)], input_length=maxlen, name='embedding')(main_input) embedding = Dropout(0.50)(embedding) conv4 = Convolution1D(nb_filter=nb_filter, filter_length=4, border_mode='valid', activation='relu', subsample_length=1, name='conv4')(embedding) maxConv4 = MaxPooling1D(pool_length=2, name='maxConv4')(conv4) conv5 = Convolution1D(nb_filter=nb_filter, filter_length=5, border_mode='valid', activation='relu', subsample_length=1, name='conv5')(embedding) maxConv5 = MaxPooling1D(pool_length=2, name='maxConv5')(conv5) x = merge([maxConv4, maxConv5], mode='concat') x = Dropout(0.15)(x) x = RNN(rnn_output_size)(x) x = Dense(hidden_dims, activation='relu', init='he_normal', W_constraint = maxnorm(3), b_constraint=maxnorm(3), name='mlp')(x) x = Dropout(0.10, name='drop')(x) output = Dense(1, init='he_normal', activation='sigmoid', name='output')(x) model = Model(input=main_input, output=output) model.compile(loss={'output':'binary_crossentropy'}, optimizer=Adadelta(lr=0.95, epsilon=1e-06), metrics=["accuracy"]) return model
def build_dense_layer(config, n_hidden=None, activation='linear'): if n_hidden is None: n_hidden = config.n_hidden return Dense(n_hidden, W_regularizer=l2(config.l2_penalty), W_constraint=maxnorm(config.dense_max_norm), activation=activation)
def build_model(self, n_classes, max_words, w2v_size, vocab_size, filter_sizes, n_filters, dense_layer_sizes, activation_function, dropout_p): positive_node, positive_input = self._build_conv_node(activation_function, max_words, w2v_size, vocab_size, n_filters, filter_sizes) negative_node, negative_input = self._build_conv_node(activation_function, max_words, w2v_size, vocab_size, n_filters, filter_sizes) merged_dense_layer = merge([[positive_node, negative_node], 'concat']) dense_layers = [] first_dense_layer = Dense(dense_layer_sizes.pop(0))(merged_dense_layer) dense_layers.append(first_dense_layer) if len(dense_layer_sizes) > 1: i = 1 for dense_layer_size in dense_layer_sizes: dense_layer = Dense(dense_layer_size, W_constraint=maxnorm(2))(dense_layer_sizes[i-1]) dense_activation_layer = Activation(activation_function)(dense_layer) dropout_layer = Dropout(dropout_p)(dense_activation_layer) dense_layers.append(dropout_layer) i += 1 # Add last layer softmax_dense_layer = Dense(n_classes)(dense_layers[-1]) softmax_layer = Activation('softmax')(softmax_dense_layer) model = Model(input=[positive_input, negative_input], output=softmax_layer) return model
def make_model(self,len_feature): ##########################Parameters for the model and dataset #TRAINING_SIZE = len(inputs) # Try replacing JZS1 with LSTM, GRU, or SimpleRNN HIDDEN_SIZE = self.node0 RNN = recurrent.LSTM(HIDDEN_SIZE, input_shape=(None, len(self.chars)), return_sequences=False,kernel_regularizer=l2(self.l2_c), bias_regularizer=l2(self.l2_c),recurrent_dropout=self.drop_out_c, dropout=self.drop_out_c) #len0_hla = 34 #ratio_t = 1 ###class number = binder or non-binder (1 = binder, 0 = non-binder) #classes = [0,1] ##########################start a model########################## ##########fixed part model_fixed = Sequential() model_fixed.add(Dense(help_nn,input_dim=len_feature, activation=act_fun, kernel_constraint=maxnorm(constrain_max))) model_fixed.add(Dropout(drop_out_c)) ##########recurrent part model_r = Sequential() if mask0: model_r.add(Masking(mask_value=0., input_shape=(MAXLEN, len(dict_aa['A'])))) model_r.add(RNN) ####merge merged = Merge([model_fixed, model_r],mode='concat') ###final final_model = Sequential() final_model.add(merged) for _ in range(0,help_layer0): final_model.add(Dense(help_nn, kernel_constraint=maxnorm(constrain_max))) final_model.add(Activation(act_fun)) final_model.add(Dropout(drop_out_c)) final_model.add(Dense(1)) final_model.compile(loss=loss_function0, optimizer="adam") model = final_model json_string = model.to_json() open(path_save+file_name0+out_name+'_model.json', 'w').write(json_string) return model
def build_convolutional_context_model(config, n_classes): graph = Graph() prev_non_word_layer = build_char_model(graph, config) # Word-level input for the context of the non-word error. graph.add_input(config.context_input_name, input_shape=(config.context_input_width,), dtype='int') context_embedding = build_embedding_layer(config, input_width=config.context_input_width, n_embeddings=config.n_context_embeddings, n_embed_dims=config.n_context_embed_dims) graph.add_node(context_embedding, name='context_embedding', input=config.context_input_name) context_conv = build_convolutional_layer(config, n_filters=config.n_context_filters, filter_width=config.context_filter_width) context_conv.trainable = config.train_filters graph.add_node(context_conv, name='context_conv', input='context_embedding') context_prev_layer = add_bn_relu(graph, config, 'context_conv') context_pool = build_pooling_layer(config, input_width=config.context_input_width, filter_width=config.context_filter_width) graph.add_node(context_pool, name='context_pool', input=context_prev_layer) graph.add_node(Flatten(), name='context_flatten', input='context_pool') prev_context_layer = 'context_flatten' prev_layer = build_fully_connected_layers(graph, config, prev_non_word_layer, prev_context_layer) prev_layer = build_residual_blocks(graph, config, prev_layer) if hasattr(config, 'n_hsm_classes'): graph.add_node(build_hierarchical_softmax_layer(config), name='softmax', input=prev_layer) else: graph.add_node(Dense(n_classes, init=config.softmax_init, W_constraint=maxnorm(config.softmax_max_norm)), name='softmax', input=prev_layer) prev_layer = 'softmax' if config.batch_normalization: graph.add_node(BatchNormalization(), name='softmax_bn', input='softmax') prev_layer = 'softmax_bn' graph.add_node(Activation('softmax'), name='softmax_activation', input=prev_layer) graph.add_output(name='multiclass_correction_target', input='softmax_activation') load_weights(config, graph) optimizer = build_optimizer(config) graph.compile(loss={'multiclass_correction_target': config.loss}, optimizer=optimizer) return graph
def buildConvolution(self, name): filters = self.params.get('filters') nb_filter = self.params.get('nb_filter') assert filters assert nb_filter convs = [] for fsz in filters: layer_name = '%s-conv-%d' % (name, fsz) conv = Convolution2D( nb_filter=nb_filter, nb_row=fsz, nb_col=self.wdim, border_mode='valid', init='glorot_uniform', W_constraint=maxnorm(self.params.get('w_maxnorm')), b_constraint=maxnorm(self.params.get('b_maxnorm')), name=layer_name ) convs.append(conv) self.layers['%s-convolution' % name] = convs
def build_embedding_layer(config, input_width=None): try: n_embeddings = config.n_vocab except AttributeError: n_embeddings = config.n_embeddings try: input_width = config.input_width except AttributeError: input_width = input_width try: mask_zero = config.mask_zero except AttributeError: mask_zero = False if hasattr(config, 'embedding_weights') and config.embedding_weights is not None: W = np.load(config.embedding_weights) if config.train_embeddings is True or config.train_embeddings == 'true': return Embedding(n_embeddings, config.n_embed_dims, weights=[W], input_length=input_width, W_constraint=maxnorm(config.embedding_max_norm), mask_zero=mask_zero) else: return ImmutableEmbedding(n_embeddings, config.n_embed_dims, weights=[W], mask_zero=mask_zero, input_length=input_width) else: if config.train_embeddings is True: return Embedding(n_embeddings, config.n_embed_dims, init=config.embedding_init, W_constraint=maxnorm(config.embedding_max_norm), mask_zero=mask_zero, input_length=input_width) else: return ImmutableEmbedding(n_embeddings, config.n_embed_dims, init=config.embedding_init, mask_zero=mask_zero, input_length=input_width)
def double_conv_layerunet(x, size, dropout, batch_norm): kernel_size=(3,3) conv = Conv2D(size,kernel_size, activation='relu',kernel_constraint=maxnorm(4.),padding='same')(x) # conv = Conv2D(size,kernel_size,kernel_constraint=maxnorm(4.,axis=-1),padding='same')(x) if batch_norm == True: conv = BatchNormalization( )(conv) # conv = Activation('relu')(conv) # conv = LeakyReLU(alpha=0.15)(conv) # if dropout > 0: # conv = Dropout(dropout)(conv) conv = Conv2D(size, kernel_size,activation='relu',kernel_constraint=maxnorm(4.), padding='same')(conv) # conv = Conv2D(size,kernel_size,kernel_constraint=maxnorm(4.,axis=-1),padding='same')(conv) if batch_norm == True: conv = BatchNormalization()(conv) # conv = Activation('relu')(conv) # conv = LeakyReLU(alpha=0.15)(conv) if dropout > 0: conv = Dropout(dropout)(conv) return conv
def create_model(self): ''' Create new keras model. ''' self.class_weight = {0: 0.25, 1: 0.75} model = Sequential() model.add(Dense(self.data.shape[1], input_dim=self.data.shape[1], activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.5)) model.add(Dense(1, activation='sigmoid')) model.compile(optimizer='RMSprop', loss='binary_crossentropy', metrics=['accuracy']) return model
def train_ensemble(X_train, X_test, Y_train, n_models=6): n_categs = Y_train.shape[1] for i in range(n_models): print('---' * 20) print('Training model #: {}'.format(i + 1)) print('---' * 20) model = Sequential() dim = random.choice(np.arange(512, 769)) model.add(Dense(output_dim=dim, input_dim=X_train.shape[1], init="glorot_uniform", W_constraint=maxnorm(1))) model.add(PReLU()) model.add(Dropout(0.6)) model.add(Dense(output_dim=n_categs, init="glorot_uniform", W_constraint=maxnorm(1))) model.add(Activation("softmax")) model.compile(loss='categorical_crossentropy', optimizer='adagrad') print("Training model...") epoch = random.choice(np.arange(100, 400)) model.fit(X_train, Y_train, nb_epoch=epoch, batch_size=512) if i == 0: probs = model.predict_proba(X_test, batch_size=512) else: probs += model.predict_proba(X_test, batch_size=512) probs /= n_models column_names, index = get_extra() df_out = pd.DataFrame(data=np.c_[index, probs], columns=column_names) df_out['VisitNumber'] = np.int32(df_out['VisitNumber']) df_out.to_csv('nnEnsemble.csv', index=False) return df_out
def model(): model=Sequential() model.add(Convolution2D(32,3,3,activation='relu',input_shape=(3,32,32),border_mode='same',W_constraint=maxnorm(3))) #model.add(Dropout(0.2)) model.add(Convolution2D(32,3,3,activation='relu',input_shape=(3,32,32),border_mode='same',W_constraint=maxnorm(3))) model.add(MaxPooling2D(pool_size=(2,2),strides=(2,2))) model.add(Flatten()) model.add(Dense(512,activation='relu',W_constraint=maxnorm(3))) #model.add(Dropout(0.5)) model.add(Dense(10,activation='softmax')) return model
def double_conv_layerunet(x, size, dropout, batch_norm,dim_org,ke_i,kernel_siz): if K.image_dim_ordering() == 'th': axis = 1 else: axis = 3 conv = Conv2D(size,kernel_siz, activation='relu',kernel_initializer=ke_i, data_format=dim_org,kernel_constraint=maxnorm(4.),padding='same')(x) if batch_norm == True: conv = BatchNormalization(axis=axis)(conv) if dropout > 0: conv = Dropout(dropout)(conv) conv = Conv2D(size, kernel_siz,activation='relu',kernel_initializer=ke_i, data_format=dim_org,kernel_constraint=maxnorm(4.), padding='same')(conv) if batch_norm == True: conv = BatchNormalization(axis=axis)(conv) # conv = LeakyReLU(alpha=0.15)(conv) return conv
def create_model(self): ''' Create new keras model. ''' self.class_weight = {0: 0.25, 1: 0.75} # Entity branch entity_inputs = Input(shape=(self.data[0].shape[1],)) entity_x = Dense(self.data[0].shape[1], activation='relu', kernel_constraint=maxnorm(3))(entity_inputs) entity_x = Dropout(0.25)(entity_x) # entity_x = Dense(50, activation='relu', # self.kernel_constraint=maxnorm(3))(entity_x) # entity_x = Dropout(0.25)(entity_x) # Candidate branch candidate_inputs = Input(shape=(self.data[1].shape[1],)) candidate_x = Dense(self.data[1].shape[1], activation='relu', kernel_constraint=maxnorm(3))(candidate_inputs) candidate_x = Dropout(0.25)(candidate_x) # candidate_x = Dense(50, activation='relu', # kernel_constraint=maxnorm(3))(candidate_x) # candidate_x = Dropout(0.25)(candidate_x) # Cosine proximity # cos_x = dot([entity_x, candidate_x], axes=1, normalize=False) # cos_x = concatenate([entity_x, candidate_x]) # cos_output = Dense(1, activation='sigmoid')(cos_x) # Match branch match_inputs = Input(shape=(self.data[2].shape[1],)) match_x = Dense(self.data[1].shape[1], activation='relu', kernel_constraint=maxnorm(3))(match_inputs) match_x = Dropout(0.25)(match_x) # Merge x = concatenate([entity_x, candidate_x, match_x]) x = Dense(32, activation='relu', kernel_constraint=maxnorm(3))(x) x = Dropout(0.25)(x) x = Dense(16, activation='relu', kernel_constraint=maxnorm(3))(x) x = Dropout(0.25)(x) x = Dense(8, activation='relu', kernel_constraint=maxnorm(3))(x) x = Dropout(0.25)(x) predictions = Dense(1, activation='sigmoid')(x) model = Model(inputs=[entity_inputs, candidate_inputs, match_inputs], outputs=predictions) model.compile(optimizer='RMSprop', loss='binary_crossentropy', metrics=['accuracy']) return model
def build_model(train_features, neurons = 100, layers = 1, verbose = 0, lmbda = 1e-5, learn_rate = 0.01, b_maxnorm = 0, batch_norm = 0, decay = 0, momentum = 0.5, dropout_P=0.5, activation = "relu"): #Start from empty sequential model where we can stack layers model = Sequential() #Add one+ fully-connected layers for l in range(layers): if l ==0: model.add(Dense(output_dim=neurons, input_dim=train_features.shape[1],W_regularizer=l1(lmbda),b_constraint=maxnorm(m=b_maxnorm))) else: model.add(Dense(b_constraint=maxnorm(m=b_norm),output_dim=neurons, input_dim=neurons,W_regularizer=l1(lmbda))) # Add activation function to each neuron model.add(Activation(activation)) #rectified linear activation #batch normalization: maintain mean activation ~ 0 and activation standard deviation ~ 1. if batch_norm: model.add(BatchNormalization()) #dropout model.add(Dropout(dropout_P)) #Add fully-connected layer with 2 neurons, one for each class of labels model.add(Dense(output_dim=2)) #Add softmax layer to force the 2 outputs to sum up to one so that we have a probability representation over the labels. model.add(Activation("softmax")) #Compile model with categorical_crossentrotry as loss, and stochastic gradient descent(learning rate=0.001, momentum=0.5 as optimizer) model.compile(loss='categorical_crossentropy', optimizer=SGD(lr= learn_rate, momentum=momentum,decay = decay, nesterov=True), metrics=['accuracy']) return model
def sk3(num_class,num_bit,img_rows,img_cols,INP_SHAPE,dim_org): print INP_SHAPE kernel_size=(3,3) print 'sk3 with num_class :',num_class model = Sequential() model.add(Conv2D(16, kernel_size, input_shape=INP_SHAPE, padding='same', activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3))) model.add(Conv2D(16, kernel_size, padding='same', activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3))) # model.add(AveragePooling2D(pool_size=(2, 2),data_format=dim_org)) model.add(Dropout(0.1)) model.add(Conv2D(32, kernel_size, padding='same', activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3))) model.add(AveragePooling2D(pool_size=(2, 2),data_format=dim_org)) model.add(Dropout(0.2)) model.add(Conv2D(64, kernel_size, padding='same', activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3))) model.add(AveragePooling2D(pool_size=(2, 2),data_format=dim_org)) # model.add(Dropout(0.2)) # model.add(Conv2D(128, kernel_size, padding='same', # activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3))) # model.add(MaxPooling2D(pool_size=(2, 2),data_format=dim_org)) model.add(Dropout(0.3)) model.add(Conv2D(128, kernel_size, padding='same', activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3))) model.add(UpSampling2D(size=(2, 2),data_format=dim_org)) model.add(Dropout(0.3)) model.add(Conv2DTranspose(64, kernel_size, padding='same', activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3))) model.add(UpSampling2D(size=(2, 2),data_format=dim_org)) model.add(Dropout(0.2)) model.add(Conv2DTranspose(32, kernel_size, padding='same', activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3))) # model.add(UpSampling2D(size=(2, 2),data_format=dim_org)) model.add(Dropout(0.1)) model.add(Conv2DTranspose(num_class, kernel_size, activation='softmax', data_format=dim_org,padding='same')) return model
def build_residual_blocks(graph, config, prev_layer): # Add sequence of residual blocks. for i in range(config.n_residual_blocks): # Add a fixed number of layers per residual block. block_name = '%02d' % i graph.add_node(Identity(), name=block_name+'input', input=prev_layer) prev_layer = block_input_layer = block_name+'input' try: n_layers_per_residual_block = config.n_layers_per_residual_block except AttributeError: n_layers_per_residual_block = 2 for layer_num in range(n_layers_per_residual_block): layer_name = 'h%s%02d' % (block_name, layer_num) l = Dense(config.n_hidden_residual, init=config.residual_init, W_constraint=maxnorm(config.residual_max_norm)) graph.add_node(l, name=layer_name, input=prev_layer) prev_layer = layer_name if config.batch_normalization: graph.add_node(BatchNormalization(), name=layer_name+'bn', input=prev_layer) prev_layer = layer_name+'bn' if i < n_layers_per_residual_block: graph.add_node(Activation('relu'), name=layer_name+'relu', input=prev_layer) prev_layer = layer_name+'relu' if config.dropout_fc_p > 0.: graph.add_node(Dropout(config.dropout_fc_p), name=layer_name+'do', input=prev_layer) prev_layer = layer_name+'do' graph.add_node(Identity(), name=block_name+'output', inputs=[block_input_layer, prev_layer], merge_mode='sum') graph.add_node(Activation('relu'), name=block_name+'relu', input=block_name+'output') prev_layer = block_input_layer = block_name+'relu' return prev_layer
def __init__(self, vocab_size, nb_labels, emb_dim, maxlen, embedding_weights, filter_hs, nb_filters, dropout_p, trainable_embeddings, pretrained_embeddings): self.nb_labels = nb_labels if pretrained_embeddings is False: embedding_weights = None else: embedding_weights = [embedding_weights] sentence_input = Input(shape=(maxlen,), dtype='int32') x = Embedding(input_dim=vocab_size+1, output_dim=emb_dim, input_length=maxlen, dropout=dropout_p[0], weights=embedding_weights, trainable=trainable_embeddings) x = x(sentence_input) conv_pools = [] for filter_h in filter_hs: conv = Convolution1D(nb_filters, filter_h, border_mode='same', W_constraint=maxnorm(2)) conved = conv(x) batchnorm = BatchNormalization()(conved) #conved_relu = Activation('relu')(batchnorm) conved_relu = PReLU()(batchnorm) pool = Lambda(max_1d, output_shape=(nb_filters,)) pooled = pool(conved_relu) conv_pools.append(pooled) if len(conv_pools) == 1: merged = conv_pools[0] else: merged = merge(conv_pools, mode='concat') dropout = Dropout(dropout_p[1])(merged) if nb_labels == 2: out = Dense(1, activation='sigmoid') out = out(dropout) else: out = Dense(nb_labels, activation='softmax') out = out(dropout) self.model = Model(input=sentence_input, output=out)
#model model = Sequential() model.add(Embedding(maxDictionary_size, 32, input_length=maxWordCount)) #to change words to ints # model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')) # model.add(MaxPooling1D(pool_size=2)) # model.add(Dropout(0.5)) # model.add(Conv1D(filters=32, kernel_size=2, padding='same', activation='relu')) # model.add(MaxPooling1D(pool_size=2)) #hidden layers model.add(LSTM(10)) # model.add(Flatten()) model.add(Dropout(0.6)) model.add(Dense(1200, activation='relu', W_constraint=maxnorm(1))) # model.add(Dropout(0.6)) model.add(Dense(500, activation='relu', W_constraint=maxnorm(1))) # model.add(Dropout(0.5)) #output layer model.add(Dense(5, activation='softmax')) # Compile model # adam=Adam(lr=learning_rate, beta_1=0.7, beta_2=0.999, epsilon=1e-08, decay=0.0000001) model.summary() learning_rate = 0.0001 epochs = 60 batch_size = 32 #32
# #optimizer # # sgd = optimizers.SGD(lr=0.01,momentum=True) # # compile model # model.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy']) # return model left_branch = Sequential() left_branch.add( Convolution2D(nb_filter, nb_row, nb_col, border_mode='same', input_shape=(np.array(dataX).shape[1], np.array(dataX).shape[2], np.array(dataX).shape[3]), W_constraint=maxnorm(1), init='normal')) left_branch.add( MaxPooling2D(pool_size=(pool_size, pool_size), border_mode='same')) left_branch.add(Activation('relu')) left_branch.add(Flatten()) right_branch = Sequential() right_branch.add(Dense(32, input_dim=n_frames * n_hours)) merged = Merge([left_branch, right_branch], mode='concat') model = Sequential() model.add(merged) model.add(Dense(np.array(dataY).shape[1]))
from keras.constraints import maxnorm from keras.optimizers import SGD def createSupervisedEncoder(X1,X2,Y, learning_rate=0.1, ENCODING_DIM = 5 other_parameters = []): # Remark : input_shape can be (x,None) to allow input to have a variable dimension in the None part branch1 = Sequential() branch1.add(Dense(X1.shape[1], input_shape = (X1.shape[1],), init = 'normal', activation = 'relu')) branch2 = Sequential() branch2.add(Dense(X2.shape[1], input_shape = (X2.shape[1],), init = 'normal', activation = 'relu')) branch2.add(Dense(X2.shape[1], init = 'normal', activation = 'relu', W_constraint = maxnorm(5))) model = Sequential() model.add(Merge([branch1, branch2], mode = 'concat')) model.add(Dense(ENCODING_DIM, init = 'normal', activation = 'sigmoid')) # Use a concatenate layer instead ? # TODO Remark : you can also write layers as : LayerType(layer_parms)(previous_layer_object) model.compile(loss = 'binary_crossentropy', optimizer = SGD(lr = learning_rate), metrics = ['accuracy']) seed(42) model.fit([X1, X2], Y.values, batch_size = 2000, nb_epoch = 100, verbose = 1)
X_train = X_train / 255.0 X_test = X_test / 255.0 y_train = np_utils.to_categorical(y_train) y_test = np_utils.to_categorical(y_test) num_classes = y_test.shape[1] print('Classes - ', num_classes) model = Sequential() model.add( Conv2D(32, (3, 3), input_shape=(3, 32, 32), padding='same', activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.2)) model.add( Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(512, activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) epochs = 25 lrate = 0.01 decay = lrate / epochs
def catdog(): # 학습 관련 파라메타들 sizeOffm = 4 sizeOfpm = 3 my_maxnorm = 2. model = Sequential() # model.add(Convolution2D(16, 5, 5, border_mode='same', input_shape=(3, ROWS, COLS), kernel_constraint=maxnorm(2), activation='relu')) model.add( Convolution2D(16, sizeOffm, sizeOffm, border_mode='same', input_shape=(3, ROWS, COLS), kernel_constraint=maxnorm(my_maxnorm))) model.add(BatchNormalization()) #model.add(Activation('relu')) model.add(Activation(LeakyReLU(alpha=0.1))) model.add( Convolution2D(16, sizeOffm, sizeOffm, border_mode='same', input_shape=(3, ROWS, COLS), kernel_constraint=maxnorm(my_maxnorm))) model.add(BatchNormalization()) #model.add(Activation('relu')) model.add(Activation(LeakyReLU(alpha=0.1))) model.add( MaxPooling2D(pool_size=(sizeOfpm, sizeOfpm), dim_ordering="th")) model.add( Convolution2D(32, sizeOffm, sizeOffm, border_mode='same', input_shape=(3, ROWS, COLS), kernel_constraint=maxnorm(my_maxnorm))) model.add(BatchNormalization()) #model.add(Activation('relu')) model.add(Activation(LeakyReLU(alpha=0.1))) model.add( Convolution2D(32, sizeOffm, sizeOffm, border_mode='same', input_shape=(3, ROWS, COLS), kernel_constraint=maxnorm(my_maxnorm))) model.add(BatchNormalization()) #model.add(Activation('relu')) model.add(Activation(LeakyReLU(alpha=0.1))) model.add( MaxPooling2D(pool_size=(sizeOfpm, sizeOfpm), dim_ordering="th")) model.add( Convolution2D(64, sizeOffm, sizeOffm, border_mode='same', input_shape=(3, ROWS, COLS), kernel_constraint=maxnorm(my_maxnorm))) model.add(BatchNormalization()) #model.add(Activation('relu')) model.add(Activation(LeakyReLU(alpha=0.1))) model.add( Convolution2D(64, sizeOffm, sizeOffm, border_mode='same', input_shape=(3, ROWS, COLS), kernel_constraint=maxnorm(my_maxnorm))) model.add(BatchNormalization()) #model.add(Activation('relu')) model.add(Activation(LeakyReLU(alpha=0.1))) model.add( MaxPooling2D(pool_size=(sizeOfpm, sizeOfpm), dim_ordering="th")) model.add( Convolution2D(128, sizeOffm, sizeOffm, border_mode='same', input_shape=(3, ROWS, COLS), kernel_constraint=maxnorm(my_maxnorm))) model.add(BatchNormalization()) #model.add(Activation('relu')) model.add(Activation(LeakyReLU(alpha=0.2))) model.add( Convolution2D(128, sizeOffm, sizeOffm, border_mode='same', input_shape=(3, ROWS, COLS), kernel_constraint=maxnorm(my_maxnorm))) model.add(BatchNormalization()) #model.add(Activation('relu')) model.add(Activation(LeakyReLU(alpha=0.2))) model.add( MaxPooling2D(pool_size=(sizeOfpm, sizeOfpm), dim_ordering="th")) model.add(Flatten()) # model.add(Dense(256, kernel_constraint=maxnorm(my_maxnorm), activation='relu')) model.add( MaxoutDense(output_dim=128, nb_feature=8, init='glorot_uniform')) #model.add(Dropout(0.3)) # model.add(Dense(256, kernel_constraint=maxnorm(my_maxnorm), activation='relu')) model.add( MaxoutDense(output_dim=128, nb_feature=8, init='glorot_uniform')) #model.add(Dropout(0.3)) model.add(Dense(1)) model.add(Activation('sigmoid')) # model.compile(loss=objective, optimizer=optimizer, metrics=['accuracy']) model.compile(loss=objective, optimizer=optimizer, metrics=['accuracy']) return model
# Create the model model = Sequential() model.add(Conv2D(32, (3, 3), activation='linear', padding='same',input_shape=train_data.shape[1:])) model.add(PReLU()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(BatchNormalization()) model.add(Conv2D(48,(3, 3), activation='linear', padding='same')) model.add(PReLU()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(BatchNormalization()) model.add(Conv2D(64, (3, 3), activation='linear', padding='same')) model.add(PReLU()) model.add(Dropout(0.1)) model.add(Flatten()) model.add(BatchNormalization()) model.add(Dense(1024, activation='linear', kernel_constraint=maxnorm(3))) model.add(PReLU()) model.add(Dropout(0.2)) model.add(Dense(84, activation='linear', kernel_constraint=maxnorm(3))) model.add(PReLU()) model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy',optimizer=Adam(lr=lr, decay = lr*0.9),metrics=['accuracy']) #Training the Model datagen = ImageDataGenerator(zoom_range=0.2,horizontal_flip=True) model.fit_generator(datagen.flow(train_data, train_labels, batch_size=128),steps_per_epoch=len(train_data) /128, epochs=epochs,verbose =0) #Testing the Model #Reading test data test_file = unpickle(test) test_data = test_file["data"] test_data = StandardScaler().fit_transform(test_data.astype(float)) test_data = test_data.reshape(test_data.shape[0], 3, 32, 32)
print(X_train[0]) print (len(y_train)) print (len(X_train)) print("Pad sequences (samples x time)") X_train = sequence.pad_sequences(X_train, maxlen=maxlen) X_test = sequence.pad_sequences(X_test, maxlen=maxlen) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) print('Build model...') model = Sequential() model.add(Embedding(max_features, 200, input_length=maxlen)) model.add(LSTM(300)) model.add(Dense(210,init='he_normal',W_constraint = maxnorm(2))) model.add(Activation('sigmoid')) model.add(Dense(205,init='he_normal',W_constraint = maxnorm(2))) model.add(Activation('tanh')) model.add(Dense(201,init='he_normal',W_constraint = maxnorm(2))) model.compile(loss=hinge2, optimizer='adam') print("Train...") model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=25, validation_data=(X_test, y_test)) json_string = model.to_json() open('my_model_struct.json', 'w').write(json_string) print('model structure saved') model.save_weights('my_model_weights.h5') print('model weights saved')
test, test_xy, test_x, test_y, test_set_x, test_set_y = read_dataset( test_dataset, test_data_args) # Reading dev dataset dev_dataset, dev_data_args = read_data_args(dev_data_file) dev, dev_xy, dev_x, dev_y, dev_set_x, dev_set_y = read_dataset( dev_dataset, dev_data_args) train_set_y = train_set_y.astype(numpy.int64) valid_set_y = valid_set_y.astype(numpy.int64) test_set_y = test_set_y.astype(numpy.int64) dev_set_y = dev_set_y.astype(numpy.int64) model = Sequential() model.add(Dropout(0.2, input_shape=(440, ))) model.add(Dense(1024, init='uniform', W_constraint=maxnorm(4))) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(1024, init='uniform', W_constraint=maxnorm(4))) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(1024, init='uniform', W_constraint=maxnorm(4))) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(1024, init='uniform', W_constraint=maxnorm(4))) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(1940, init='uniform')) model.add(Activation('softmax')) model.load_weights('weights3.h5')
def DLS2F_construct_withaa_complex_win_filter_layer_opt(win_array,ktop_node,output_dim,use_bias,hidden_type,nb_filters,nb_layers,opt,hidden_num): ss_feature_num = 3 aa_feature_num = 20 ktop_node= ktop_node win_array = list(win_array) print("Setting hidden models as ",hidden_type) print("Setting nb_filters as ",nb_filters) print("Setting nb_layers as ",nb_layers) print("Setting opt as ",opt) print("Setting win_array as ",win_array) print("Setting use_bias as ",use_bias) ########################################## set up model DLS2F_input_shape =(None,aa_feature_num+ss_feature_num) filter_sizes=win_array DLS2F_input = Input(shape=DLS2F_input_shape) DLS2F_convs = [] for fsz in filter_sizes: DLS2F_conv = DLS2F_input for i in range(0,nb_layers): DLS2F_conv = _conv_bn_relu1D(nb_filter=nb_filters, nb_row=fsz, subsample=1,use_bias=use_bias)(DLS2F_conv) DLS2F_pool = K_max_pooling1d(ktop=ktop_node)(DLS2F_conv) DLS2F_flatten = Flatten()(DLS2F_pool) DLS2F_convs.append(DLS2F_flatten) if len(filter_sizes)>1: DLS2F_out = Merge(mode='concat')(DLS2F_convs) else: DLS2F_out = DLS2F_convs[0] DLS2F_dense1 = Dense(output_dim=hidden_num, init='he_normal', activation=hidden_type, W_constraint=maxnorm(3))(DLS2F_out) # changed on 20170314 to check if can visualzie better DLS2F_dropout1 = Dropout(0.2)(DLS2F_dense1) DLS2F_output = Dense(output_dim=output_dim, init="he_normal", activation="softmax")(DLS2F_dropout1) DLS2F_ResCNN = Model(input=[DLS2F_input], output=DLS2F_output) DLS2F_ResCNN.compile(loss="categorical_crossentropy", metrics=['accuracy'], optimizer=opt) return DLS2F_ResCNN
def run(model_name, dataset_name): """ run the baseline model :param model_name: name of baseline models, CNN or LSTM :param dataset_name: name of datasets, candidate values [bbc, digg, MySpace, rw, Twitter, YouTube] """ print "model: %s" % model_name print "process dataset %s..." % dataset_name data = cPickle.load(open('./pkl/%s.pkl' % dataset_name, 'rb')) records, glove_embeddings, vocab, word_to_df = data dim_emb = len(glove_embeddings[1]) # index of word starts from 1 # initial weights of embedding layer embeddings = np.zeros((len(vocab) + 1, dim_emb), dtype='float32') for w in vocab: wid = vocab[w] embeddings[wid, :] = glove_embeddings[wid] train_x, train_y, val_x, val_y, test_x, test_y, test_strength = [], [], [], [], [], [], [] max_len = 0 for r in records: text = r['text'] y = r['label'] wids = [vocab[w] for w in text.split(' ')] if len(wids) > max_len: max_len = len(wids) if r['type'] == 'train': train_x.append(wids) train_y.append(y) elif r['type'] == 'val': val_x.append(wids) val_y.append(y) elif r['type'] == 'test': strength = r['strength'] test_x.append(wids) test_y.append(y) test_strength.append(strength) train_x, val_x, test_x = ToArray(train=train_x, val=val_x, test=test_x) train_y, val_y, test_y = ToArray(train=train_y, val=val_y, test=test_y) _, _, test_strength = ToArray(train=[], val=[], test=test_strength) #print train_x.shape, val_x.shape, test_x.shape train_x, val_x, test_x = Padding(train=train_x, val=val_x, test=test_x, max_len=max_len) batch_size = 50 if model_name == 'CNN' else 32 if train_x.shape[0] % batch_size: n_extra = batch_size - train_x.shape[0] % batch_size x_extra = train_x[:n_extra, :] y_extra = train_y[:n_extra] train_x = np.append(train_x, x_extra, axis=0) train_y = np.append(train_y, y_extra, axis=0) np.random.seed(38438) # shuffle the training set train_set = np.random.permutation(zip(train_x, train_y)) train_x, train_y = [], [] for (x, y) in train_set: train_x.append(x) train_y.append(y) n_labels = 2 train_x = np.array(train_x) train_y = np.array(train_y) train_y = to_categorical(train_y) val_y = to_categorical(val_y) print "n_train: %s, n_val: %s, n_test: %s" % ( train_x.shape[0], val_x.shape[0], test_x.shape[0]) if model_name == 'CNN': model = Graph() model.add_input(name='input', input_shape=(max_len, ), dtype='int') model.add_node(Embedding(input_dim=len(vocab) + 1, output_dim=dim_emb, input_length=max_len, weights=[embeddings]), name="emb", input="input") filter_hs = [3, 4, 5] n_filter = 100 dropout_rate = 0.5 n_epoch = 20 for i in xrange(len(filter_hs)): win_size = filter_hs[i] conv_name = 'conv%s' % i pool_name = "pool%s" % i flatten_name = "flatten%s" % i pool_size = max_len - win_size + 1 model.add_node( layer=Convolution1D(nb_filter=n_filter, filter_length=win_size, activation='relu', W_constraint=maxnorm(m=3), b_constraint=maxnorm(m=3)), name=conv_name, input='emb', ) model.add_node(layer=MaxPooling1D(pool_length=pool_size), name=pool_name, input=conv_name) model.add_node(layer=Flatten(), name=flatten_name, input=pool_name) model.add_node(layer=Dropout(p=dropout_rate), name="dropout", inputs=["flatten0", "flatten1", "flatten2"]) model.add_node(layer=Dense(output_dim=n_labels, activation='softmax'), name='softmax', input='dropout') model.add_output(input='softmax', name="output") model.compile(loss={'output': 'categorical_crossentropy'}, optimizer='adadelta', metrics=['accuracy']) model_path = './model/%s_%s.hdf5' % (model_name, dataset_name) best_model = ModelCheckpoint(filepath=model_path, monitor='val_acc', save_best_only=True, mode='max') print "training..." model.fit(data={ 'input': train_x, 'output': train_y }, batch_size=batch_size, nb_epoch=n_epoch, validation_data={ 'input': val_x, 'output': val_y }, callbacks=[best_model], verbose=0) else: model = Sequential() model.add( Embedding(input_dim=len(vocab) + 1, output_dim=dim_emb, mask_zero=True, input_length=max_len, weights=[embeddings])) model.add(LSTM(output_dim=128, dropout_W=0.2, dropout_U=0.2)) model.add(Dense(n_labels, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model_path = './model/%s_%s.hdf5' % (model_name, dataset_name) best_model = ModelCheckpoint(filepath=model_path, monitor='val_acc', save_best_only=True, mode='max') n_epoch = 20 print "training..." model.fit(x=train_x, y=train_y, batch_size=batch_size, nb_epoch=n_epoch, validation_data=(val_x, val_y), callbacks=[best_model], verbose=0) pred_strength = [] print "load the best model from disk..." model.load_weights(model_path) if model_name == 'LSTM': pred_strength = model.predict(x=test_x, batch_size=batch_size) else: for i in xrange(len(test_x)): res = model.predict(data={'input': test_x[i:i + 1]}, batch_size=1) pred_strength.append(res['output']) pred_strength = np.array(pred_strength) pred_strength = pred_strength.reshape( (pred_strength.shape[0], pred_strength.shape[2])) assert pred_strength.shape == test_strength.shape print "evaluate performance of the system..." accu, mae, rmse = evaluate(strength_gold=test_strength, strength_pred=pred_strength) print "%s over %s--->accuracy: %s, mae: %s, rmse: %s\n\n" % ( model_name, dataset_name, accu, mae, rmse) pred_strengths_lines = [] for strength in pred_strength: pred_strengths_lines.append('%s\n' % ' '.join([str(ele) for ele in strength]))
model.add(Conv2D(64, (3, 3), padding='same')) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(BatchNormalization()) model.add(Conv2D(128, (3, 3), padding='same')) model.add(Activation('relu')) model.add(Dropout(0.2)) model.add(BatchNormalization()) model.add(Flatten()) model.add(Dropout(0.2)) model.add(Dense(256, kernel_constraint=maxnorm(3))) model.add(Activation('relu')) model.add(Dropout(0.2)) model.add(BatchNormalization()) model.add(Dense(128, kernel_constraint=maxnorm(3))) model.add(Activation('relu')) model.add(Dropout(0.2)) model.add(BatchNormalization()) model.add(Dense(class_num)) model.add(Activation('softmax')) epochs = 25 optimizer = 'adam'
model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) # model.add(BatchNormalization()) model.add(Conv2D(128, (3, 3), padding='same')) model.add(Activation('relu')) model.add(Dropout(0.2)) # model.add(BatchNormalization()) model.add(Flatten()) model.add(Dropout(0.2)) model.add( Dense(256, kernel_regularizer=regularizers.l2(0.0001), kernel_constraint=maxnorm(3))) model.add(Activation('relu')) model.add(Dropout(0.2)) # model.add(BatchNormalization()) model.add( Dense(128, kernel_regularizer=regularizers.l2(0.0001), kernel_constraint=maxnorm(3))) model.add(Activation('relu')) model.add(Dropout(0.2)) # model.add(BatchNormalization()) model.add(Dense(num_classes)) model.add(Activation('softmax')) epochs = 76 optimizer = 'Adam'
def DLS2F_construct_withaa_complex_win_filter_layer_opt( win_array, ktop_node, output_dim, use_bias, hidden_type, nb_filters, nb_layers, opt, hidden_num): ss_feature_num = 3 sa_feature_num = 2 aa_feature_num = 20 pssm_feature_num = 20 ktop_node = ktop_node print "Setting hidden models as ", hidden_type print "Setting nb_filters as ", nb_filters print "Setting nb_layers as ", nb_layers print "Setting opt as ", opt print "Setting win_array as ", win_array print "Setting use_bias as ", use_bias ########################################## set up model if (two_stream): # TWO STREAM DLS2F_input_shape_aa = (None, aa_feature_num) DLS2F_input_shape_rest = (None, ss_feature_num + sa_feature_num + pssm_feature_num) filter_sizes = win_array DLS2F_input_aa = Input(shape=DLS2F_input_shape_aa, name='input_aa') DLS2F_input_rest = Input(shape=DLS2F_input_shape_rest, name='input_rest') DLS2F_convs_aa = [] DLS2F_convs_rest = [] for fsz in filter_sizes: DLS2F_conv_aa = DLS2F_input_aa for i in range(0, nb_layers): DLS2F_conv_aa = _conv_bn_relu1D( nb_filter=nb_filters, nb_row=fsz, subsample=1, use_bias=use_bias)(DLS2F_conv_aa) if extra_fusion_CONV: DLS2F_pool_aa = K_max_pooling1d(ktop=ktop_node)(DLS2F_conv_aa) DLS2F_convs_aa.append(DLS2F_pool_aa) else: DLS2F_pool_aa = K_max_pooling1d(ktop=ktop_node)(DLS2F_conv_aa) DLS2F_flatten_aa = Flatten()(DLS2F_pool_aa) DLS2F_convs_aa.append(DLS2F_flatten_aa) # TWO STREAM for fsz in filter_sizes: DLS2F_conv_rest = DLS2F_input_rest for i in range(0, nb_layers): DLS2F_conv_rest = _conv_bn_relu1D( nb_filter=nb_filters, nb_row=fsz, subsample=1, use_bias=use_bias)(DLS2F_conv_rest) if extra_fusion_CONV: DLS2F_pool_rest = K_max_pooling1d( ktop=ktop_node)(DLS2F_conv_rest) DLS2F_convs_rest.append(DLS2F_pool_rest) else: DLS2F_pool_rest = K_max_pooling1d( ktop=ktop_node)(DLS2F_conv_rest) DLS2F_flatten_rest = Flatten()(DLS2F_pool_rest) DLS2F_convs_rest.append(DLS2F_flatten_rest) if extra_fusion_FC: # TWO STREAM if len(filter_sizes) > 1: DLS2F_out_aa = Merge(mode='concat')(DLS2F_convs_aa) DLS2F_out_rest = Merge(mode='concat')(DLS2F_convs_rest) DLS2F_dense_aa_two_stream = Dense( output_dim=hidden_num, init='he_normal', activation=hidden_type, W_constraint=maxnorm(3))( DLS2F_out_aa ) # changed on 20170314 to check if can visualzie better DLS2F_dense_rest_two_stream = Dense( output_dim=hidden_num, init='he_normal', activation=hidden_type, W_constraint=maxnorm(3))( DLS2F_out_rest ) # changed on 20170314 to check if can visualzie better DLS2F_dropout_aa_two_stream = Dropout(0.2)( DLS2F_dense_aa_two_stream) DLS2F_dropout_rest_two_stream = Dropout(0.2)( DLS2F_dense_rest_two_stream) DLS2F_out = Merge(mode='concat')(DLS2F_dropout_aa_two_stream + DLS2F_dropout_rest_two_stream) else: DLS2F_out = DLS2F_convs[0] else: if len(filter_sizes) > 1: DLS2F_convs = DLS2F_convs_aa + DLS2F_convs_rest DLS2F_out = Merge(mode='concat')(DLS2F_convs) else: DLS2F_out = DLS2F_convs[0] if extra_fusion_CONV: DLS2F_conv_extra = DLS2F_out for i in range(0, nb_layers): DLS2F_conv_extra = _conv_bn_relu1D( nb_filter=nb_filters, nb_row=6, subsample=1, use_bias=use_bias)(DLS2F_conv_extra) DLS2F_pool_extra = K_max_pooling1d( ktop=ktop_node)(DLS2F_conv_extra) DLS2F_flatten_extra = Flatten()(DLS2F_pool_extra) DLS2F_dense1_two_stream = Dense( output_dim=hidden_num, init='he_normal', activation=hidden_type, W_constraint=maxnorm(3))( DLS2F_flatten_extra ) # changed on 20170314 to check if can visualzie better else: DLS2F_dense1_two_stream = Dense( output_dim=hidden_num, init='he_normal', activation=hidden_type, W_constraint=maxnorm(3))( DLS2F_out ) # changed on 20170314 to check if can visualzie better # TWO STREAM DLS2F_dropout1_two_stream = Dropout(0.2)(DLS2F_dense1_two_stream) DLS2F_output_two_stream = Dense( output_dim=output_dim, init="he_normal", activation="softmax")(DLS2F_dropout1_two_stream) DLS2F_ResCNN_two_stream = Model( input=[DLS2F_input_aa, DLS2F_input_rest], output=DLS2F_output_two_stream) DLS2F_ResCNN_two_stream.compile(loss="categorical_crossentropy", metrics=['accuracy'], optimizer=opt) # TWO STREAM DLS2F_ResCNN_two_stream.summary() return DLS2F_ResCNN_two_stream else: # DEFAULT DO NOT EDIT DLS2F_input_shape = (None, aa_feature_num + ss_feature_num + sa_feature_num + pssm_feature_num) filter_sizes = win_array DLS2F_input = Input(shape=DLS2F_input_shape) DLS2F_convs = [] if pyramid_window_size: for fsz in filter_sizes: DLS2F_conv = DLS2F_input for i in range(0, nb_layers): if pyramid_nb_filters: DLS2F_conv = _conv_bn_relu1D( nb_filter=nb_filters - i, nb_row=fsz - i, subsample=1, use_bias=use_bias)(DLS2F_conv) else: DLS2F_conv = _conv_bn_relu1D( nb_filter=nb_filters, nb_row=fsz if i == 0 else fsz / 2, subsample=1, use_bias=use_bias)(DLS2F_conv) DLS2F_pool = K_max_pooling1d(ktop=ktop_node)(DLS2F_conv) DLS2F_flatten = Flatten()(DLS2F_pool) DLS2F_convs.append(DLS2F_flatten) else: for fsz in filter_sizes: DLS2F_conv = DLS2F_input for i in range(0, nb_layers): if pyramid_nb_filters: DLS2F_conv = _conv_bn_relu1D( nb_filter=nb_filters - i, nb_row=fsz, subsample=1, use_bias=use_bias)(DLS2F_conv) else: DLS2F_conv = _conv_bn_relu1D( nb_filter=nb_filters, nb_row=fsz, subsample=1, use_bias=use_bias)(DLS2F_conv) DLS2F_pool = K_max_pooling1d(ktop=ktop_node)(DLS2F_conv) DLS2F_flatten = Flatten()(DLS2F_pool) DLS2F_convs.append(DLS2F_flatten) # DEFAULT DO NOT EDIT if len(filter_sizes) > 1: DLS2F_out = Merge(mode='concat')(DLS2F_convs) else: DLS2F_out = DLS2F_convs[0] # DEFAULT DO NOT EDIT DLS2F_dense1 = Dense( output_dim=hidden_num, init='he_normal', activation=hidden_type, W_constraint=maxnorm(3) )(DLS2F_out) # changed on 20170314 to check if can visualzie better DLS2F_dropout1 = Dropout(0.2)(DLS2F_dense1) DLS2F_output = Dense(output_dim=output_dim, init="he_normal", activation="softmax")(DLS2F_dropout1) DLS2F_ResCNN = Model(input=[DLS2F_input], output=DLS2F_output) DLS2F_ResCNN.compile(loss="categorical_crossentropy", metrics=['accuracy'], optimizer=opt) # DEFAULT DO NOT EDIT DLS2F_ResCNN.summary() return DLS2F_ResCNN
def model_define( self ): # Defines the modified sequential class with regularizers defined. model = Sequential() l2_param = 0.0 l1_param = 0.0 dropout_param = 0.0 Gaussian_noise = 1 initializer = "uniform" activation_func = "relu" adam = Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=5e-09) model.add( Dense(2048, input_dim=2048, kernel_initializer=initializer, activation=activation_func, activity_regularizer=l1_l2(l1_param, l2_param), kernel_constraint=maxnorm(3))) model.add(Dropout(dropout_param)) model.add(BatchNormalization()) model.add(GaussianNoise(Gaussian_noise)) model.add( Dense(2048, kernel_initializer=initializer, activation=activation_func, activity_regularizer=l1_l2(l1_param, l2_param), kernel_constraint=maxnorm(3))) model.add(Dropout(dropout_param)) model.add(BatchNormalization()) model.add(GaussianNoise(Gaussian_noise)) model.add( Dense(512, kernel_initializer=initializer, activation=activation_func, activity_regularizer=l1_l2(l1_param, l2_param), kernel_constraint=maxnorm(3))) model.add(Dropout(dropout_param)) model.add(BatchNormalization()) model.add(GaussianNoise(Gaussian_noise)) model.add( Dense(64, kernel_initializer=initializer, activation=activation_func, activity_regularizer=l1_l2(l1_param, l2_param), kernel_constraint=maxnorm(3))) model.add(Dropout(dropout_param)) model.add(BatchNormalization()) model.add(Dense(3, activation="softmax", kernel_initializer="normal")) model.compile(loss="categorical_crossentropy", optimizer=adam, metrics=["accuracy"]) return model
def design_dnn(nb_features, input_shape, nb_levels, conv_size, nb_labels, feat_mult=1, pool_size=2, padding='same', activation='elu', final_layer='dense-sigmoid', conv_dropout=0, conv_maxnorm=0, nb_input_features=1, batch_norm=False, name=None, prefix=None, use_strided_convolution_maxpool=True, nb_conv_per_level=2): """ "deep" cnn with dense or global max pooling layer @ end... Could use sequential... """ model_name = name if model_name is None: model_name = 'model_1' if prefix is None: prefix = model_name ndims = len(input_shape) input_shape = tuple(input_shape) convL = getattr(KL, 'Conv%dD' % ndims) maxpool = KL.MaxPooling3D if len(input_shape) == 3 else KL.MaxPooling2D if isinstance(pool_size, int): pool_size = (pool_size, ) * ndims # kwargs for the convolution layer conv_kwargs = {'padding': padding, 'activation': activation} if conv_maxnorm > 0: conv_kwargs['kernel_constraint'] = maxnorm(conv_maxnorm) # initialize a dictionary enc_tensors = {} # first layer: input name = '%s_input' % prefix enc_tensors[name] = KL.Input(shape=input_shape + (nb_input_features, ), name=name) last_tensor = enc_tensors[name] # down arm: # add nb_levels of conv + ReLu + conv + ReLu. Pool after each of first nb_levels - 1 layers for level in range(nb_levels): for conv in range(nb_conv_per_level): if conv_dropout > 0: name = '%s_dropout_%d_%d' % (prefix, level, conv) enc_tensors[name] = KL.Dropout(conv_dropout)(last_tensor) last_tensor = enc_tensors[name] name = '%s_conv_%d_%d' % (prefix, level, conv) nb_lvl_feats = nb_features * (feat_mult**level) enc_tensors[name] = convL(nb_lvl_feats, conv_size, **conv_kwargs, name=name)(last_tensor) last_tensor = enc_tensors[name] # max pool if use_strided_convolution_maxpool: name = '%s_strided_conv_%d' % (prefix, level) enc_tensors[name] = convL(nb_lvl_feats, pool_size, **conv_kwargs, name=name)(last_tensor) last_tensor = enc_tensors[name] else: name = '%s_maxpool_%d' % (prefix, level) enc_tensors[name] = maxpool(pool_size=pool_size, name=name, padding=padding)(last_tensor) last_tensor = enc_tensors[name] # dense layer if final_layer == 'dense-sigmoid': name = "%s_flatten" % prefix enc_tensors[name] = KL.Flatten(name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_dense' % prefix enc_tensors[name] = KL.Dense(1, name=name, activation="sigmoid")(last_tensor) elif final_layer == 'dense-tanh': name = "%s_flatten" % prefix enc_tensors[name] = KL.Flatten(name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_dense' % prefix enc_tensors[name] = KL.Dense(1, name=name)(last_tensor) last_tensor = enc_tensors[name] # Omittting BatchNorm for now, it seems to have a cpu vs gpu problem # https://github.com/tensorflow/tensorflow/pull/8906 # https://github.com/fchollet/keras/issues/5802 # name = '%s_%s_bn' % prefix # enc_tensors[name] = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # last_tensor = enc_tensors[name] name = '%s_%s_tanh' % prefix enc_tensors[name] = KL.Activation(activation="tanh", name=name)(last_tensor) elif final_layer == 'dense-softmax': name = "%s_flatten" % prefix enc_tensors[name] = KL.Flatten(name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_dense' % prefix enc_tensors[name] = KL.Dense(nb_labels, name=name, activation="softmax")(last_tensor) # global max pooling layer elif final_layer == 'myglobalmaxpooling': name = '%s_batch_norm' % prefix enc_tensors[name] = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_global_max_pool' % prefix enc_tensors[name] = KL.Lambda(_global_max_nd, name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_global_max_pool_reshape' % prefix enc_tensors[name] = KL.Reshape((1, 1), name=name)(last_tensor) last_tensor = enc_tensors[name] # cannot do activation in lambda layer. Could code inside, but will do extra lyaer name = '%s_global_max_pool_sigmoid' % prefix enc_tensors[name] = KL.Conv1D(1, 1, name=name, activation="sigmoid", use_bias=True)(last_tensor) elif final_layer == 'globalmaxpooling': name = '%s_conv_to_featmaps' % prefix enc_tensors[name] = KL.Conv3D(2, 1, name=name, activation="relu")(last_tensor) last_tensor = enc_tensors[name] name = '%s_global_max_pool' % prefix enc_tensors[name] = KL.GlobalMaxPooling3D(name=name)(last_tensor) last_tensor = enc_tensors[name] # cannot do activation in lambda layer. Could code inside, but will do extra lyaer name = '%s_global_max_pool_softmax' % prefix enc_tensors[name] = KL.Activation('softmax', name=name)(last_tensor) last_tensor = enc_tensors[name] # create the model model = Model(inputs=[enc_tensors['%s_input' % prefix]], outputs=[last_tensor], name=model_name) return model
img_data = np.expand_dims(img_data, axis=4) Y = np_utils.to_categorical(labels, num_classes) x, y = shuffle(img_data, Y, random_state=2) X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=2) input_shape = img_data[0].shape print(input_shape) model.add(ZeroPadding2D((1, 1), input_shape=input_shape)) model.add(Conv2D(64, (3, 3), activation='relu', kernel_constraint=maxnorm(3))) model.add(ZeroPadding2D((1, 1))) model.add(Conv2D(64, (3, 3), activation='relu', kernel_constraint=maxnorm(3))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(ZeroPadding2D((1, 1))) model.add(Conv2D(128, (3, 3), activation='relu', kernel_constraint=maxnorm(3))) model.add(ZeroPadding2D((1, 1))) model.add(Conv2D(128, (3, 3), activation='relu', kernel_constraint=maxnorm(3))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2))
(X_train, y_train), (X_test, y_test) = cifar10.load_data() # normalize inputs from 0-255 to 0.0-1.0 X_train = X_train.astype('float32') X_test = X_test.astype('float32') X_train = X_train / 255.0 X_test = X_test / 255.0 # one hot encode outputs y_train = np_utils.to_categorical(y_train) y_test = np_utils.to_categorical(y_test) num_classes = y_test.shape[1] # Create the model model = Sequential() model.add(Conv2D(32, (3, 3), input_shape=(32, 32,3), padding='same', activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.3)) model.add(BatchNormalization()) model.add(Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))) model.add(MaxPool2D(pool_size=(2, 2))) model.add(Conv2D(64, (3, 3), input_shape=(32, 32,3), padding='same', activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.5)) model.add(BatchNormalization()) model.add(Conv2D(64, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))) model.add(MaxPool2D(pool_size=(2, 2))) model.add(Conv2D(128, (3, 3), input_shape=(32, 32,3), padding='valid', activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.7)) model.add(BatchNormalization()) model.add(Conv2D(256, (3, 3), activation='relu', padding='valid', kernel_constraint=maxnorm(3))) model.add(MaxPool2D(pool_size=(2, 2))) model.add(Conv2D(512, (2, 2), activation='relu'))
def create_model(neurons=1): # create model model = Sequential() model.add(Dense(neurons, input_dim=8, kernel_initializer='uniform', activation='linear', kernel_constraint=maxnorm(4))) model.add(Dropout(0.2)) model.add(Dense(1, kernel_initializer='uniform', activation='sigmoid')) # Compile model model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) return model
# ADD THE FIRST LAYER (CONVOLUTIONAL) # Params: # 1. filters: essentially size of image; number of inputs?? # 2. kernel_size: dimensions for kernel (typically 3x3) # 3. input_shape: 3D dimensions of figure # 4. activation: type of activation funciton # 5. padding: 'same' ensures the image doesn't shrink in the convolutional layer # 6. kernel_constraint: ensures that large numbers aren't being used; scales numbers down in kernel model.add( Conv2D(filters=32, kernel_size=(3, 3), input_shape=(32, 32, 3), activation='reLu', padding='same', kernel_constraint=maxnorm(3))) # ADD THE SECOND LAYER (MAX POOLING) # Params: # 1. pool_size: finds the mac value in each n x n section of input matrix model.add(MaxPooling2D(pool_size=(2, 2))) # BEFORE PUTTING FEATURES INTO DENSE LAYER NEED TO FLATTEN model.add(Flatten) # NOW WE CAN PUT THIS INTO A DENSE LAYER # where the 'thinking' happens # Params: # 1. units: number of neurons; directly proportional to how long you want model to train # 2. activation # 3. kernel_constraint
def LSTM_network(train_path, test_path): X_train, Y_train = load_TrainData(train_path) X_test, X_test_PhraseID = load_TestData(test_path) print( '==============================The shape of Training data & Testing data ==============================' ) print("X_train shape is", X_train.shape) print("Y_train shape is", Y_train.shape) print("X_test shape is", X_test.shape) print("X_test_PhraseID shape is", X_test_PhraseID.shape) tokenizer = Tokenizer() #To create token dictionary, every element is a document(use train and test data to encure the integrity of token dictionary) tokenizer.fit_on_texts(np.concatenate((X_train, X_test), axis=0)) #calculate the word dictionary size exclude the same word tokenizer_vocabulary_size = len(tokenizer.word_index) + 1 print("Vocabulary size", tokenizer_vocabulary_size) # print(type(X_train)) # print("Word index",Tokenizer.word_index) #split the data, use 20% of whole data as validation data Split_number = 31212 Y_val = Y_train[:Split_number] X_val = X_train[:Split_number] X_train = X_train[Split_number:] Y_train = Y_train[Split_number:] #set the max word and max dictionary size for building LSTM network ( as embedding parameters) maxWord = 60 Dictionary_size = tokenizer_vocabulary_size #transform the document to vector shape, the shape type is [the number of document, the length per document] encoded_X_train = tokenizer.texts_to_sequences(X_train) encoded_X_val = tokenizer.texts_to_sequences(X_val) encoded_X_test = tokenizer.texts_to_sequences(X_test) #padding all text to same size X_train_encoded = sequence.pad_sequences(encoded_X_train, maxlen=maxWord) X_val_encoded = sequence.pad_sequences(encoded_X_val, maxlen=maxWord) X_test_encoded = sequence.pad_sequences(encoded_X_test, maxlen=maxWord) # One Hot Encoding Y_train = keras.utils.to_categorical(Y_train, 5) Y_val = keras.utils.to_categorical(Y_val, 5) #shuffling the traing Set shuffle_data(X_train_encoded, Y_train) #Build the LSTM network model = Sequential() #change words to int type model.add(Embedding(Dictionary_size, 32, input_length=maxWord)) # model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu')) # model.add(MaxPooling1D(pool_size=2)) # model.add(Dropout(0.5)) # model.add(Conv1D(filters=32, kernel_size=2, padding='same', activation='relu')) # model.add(MaxPooling1D(pool_size=2)) #hidden layers model.add(LSTM(64)) # model.add(Flatten()) model.add(Dropout(0.5)) #Freeze some Neuron to avoid overestimate model.add(Dense(1200, activation='relu', W_constraint=maxnorm(1))) # model.add(Dropout(0.6)) model.add(Dense(500, activation='relu', W_constraint=maxnorm(1))) # model.add(Dropout(0.5)) #output layer model.add(Dense(5, activation='softmax')) # Compile model model.summary() learning_rate = 0.001 epochs = 10 batch_size = 64 #32 sgd = SGD(lr=learning_rate, nesterov=True, momentum=0.7, decay=1e-4) Nadam = keras.optimizers.Nadam(lr=0.003, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004) model.compile(loss='categorical_crossentropy', optimizer=Nadam, metrics=['accuracy']) tensorboard = keras.callbacks.TensorBoard( log_dir='./logs/log_1', histogram_freq=0, write_graph=True, write_images=False) #Create the daily record checkpointer = ModelCheckpoint( filepath="./weights/weights_1.hdf5", verbose=1, save_best_only=True, monitor="val_loss") #For saving the model between the epochs reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=0, verbose=1, mode='auto', cooldown=0, min_lr=1e-6) # control the learning rate earlyStopping = EarlyStopping( monitor='val_loss', min_delta=0, patience=6, verbose=1 ) #when early stop is actived, after the number of patience epochs stop training #Loading best weights # model.load_weights("./weights/weights_19.hdf5") print( "=============================== Training =========================================" ) # uncommit this to train # tensorboard --logdir=./logs history = model.fit( X_train_encoded, Y_train, epochs=epochs, batch_size=batch_size, verbose=1, validation_data=(X_val_encoded, Y_val), callbacks=[tensorboard, reduce_lr, checkpointer, earlyStopping]) print( "=============================== Score_Accuarcy =========================================" ) scores = model.evaluate(X_val_encoded, Y_val, verbose=0) print("Accuracy: %.2f%%" % (scores[1] * 100)) print( "=============================== Predicting =========================================" ) f = open('Submission_result.csv', 'w') f.write('PhraseId,Sentiment\n') # Get the predicted data predicted = model.predict_classes(X_test_encoded, batch_size=batch_size, verbose=1) for i in range(0, X_test_PhraseID.shape[0]): f.write(str(X_test_PhraseID[i]) + "," + str(predicted[i]) + '\n') f.close() print( "================================Done===============================") # Count the epoch epoch_count = range(1, len(history.history['loss']) + 1) #visulize the training process plt.plot(epoch_count, history.history['loss'], 'r--') plt.plot(epoch_count, history.history['val_loss'], 'b-') plt.plot(epoch_count, history.history['acc'], 'gray') plt.plot(epoch_count, history.history['val_acc'], 'green') plt.legend( ['Training Loss', 'Validation Loss', 'Training Acc', 'Validation Acc']) plt.xlabel('Epoch') plt.ylabel('Loss') plt.show()
overlap=True)) # Block 6 model.add(Conv2D(512, (3, 3), padding='same')) model.add(LeakyReLU(alpha=0.3)) model.add(Conv2D(512, (3, 3), padding='same')) model.add(LeakyReLU(alpha=0.3)) model.add(Conv2D(512, (3, 3), padding='same')) model.add(LeakyReLU(alpha=0.3)) model.add( FractionalPooling2D(pool_ratio=(1, 1.25, 1.25, 1), pseudo_random=True, overlap=True)) model.add(Reshape((16, 512))) # fc layer_1 model.add(Dense(1024, kernel_constraint=maxnorm(3))) model.add(LeakyReLU(alpha=0.3)) # fc_layer_2 model.add(Dense(512, kernel_constraint=maxnorm(3))) model.add(LeakyReLU(alpha=0.3)) model.add(Flatten()) model.add(Dense(num_classes, activation='softmax')) opt = keras.optimizers.Adadelta(1, decay=1e-4) #0.1 model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) print(model.summary())
def build_convolutional_layer(config): return Convolution1D(config.n_filters, config.filter_width, W_constraint=maxnorm(config.filter_max_norm), border_mode=config.border_mode, W_regularizer=l2(config.l2_penalty))
# Create first network with Keras from keras.models import Sequential from keras.layers import Dense from keras.layers import Dropout from keras.constraints import maxnorm import numpy as np # load pima indians dataset X = np.load('../Data/X.npy') Y = X[:, 0] X = np.array(X[:, 1::], dtype=np.int32) X.shape # create model model = Sequential() #model.add(Dropout(0.2, input_shape=(8,))) model.add(Dense(50, input_dim=8, activation='relu', W_constraint=maxnorm(3))) #model.add(Dropout(0.2)) model.add(Dense(8, init='normal', activation='relu', W_constraint=maxnorm(3))) #model.add(Dropout(0.2)) model.add(Dense(1, activation='sigmoid')) # Compile model model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) # Fit the model model.fit(X, Y, nb_epoch=40, batch_size=10) # evaluate the model scores = model.evaluate(X, Y) print("%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))
x_train_scaled = np.transpose( np.asarray(x_train_scaled_list)) # Convert to numpy array # Convert min and max lists to arrays and save min_array = np.asarray(min_list) max_array = np.asarray(max_list) np.save(path + 'min_array.npy', min_array) np.save(path + 'max_array.npy', max_array) ### Create encoder ### # First layer encoder_initial = Dense(n_nodes, activation=activation_function, input_shape=(input_dim, ), kernel_constraint=maxnorm(kc_max_norm)) # Hidden layers encoder_list = [] for i in range(0, n_hidden_layers - 1): encoder_list.append( Dense(n_nodes, activation=activation_function, kernel_constraint=maxnorm(kc_max_norm))) # Final layer encoder_final = Dense(bottleneck_size, activation="linear") ### Create decoder ### # First and hidden layers decoder_list = []
def create_model(X_vocab_len, X_max_len, y_vocab_len, y_max_len, n_phonetic_features, y1, n1, y2, n2, y3, n3, y4, n4, y5, n5, y6, n6, hidden_size, num_layers): def smart_merge(vectors, **kwargs): return vectors[0] if len(vectors) == 1 else merge(vectors, **kwargs) current_word = Input(shape=(X_max_len, ), dtype='float32', name='input1') # for encoder (shared) decoder_input = Input(shape=(X_max_len, ), dtype='float32', name='input3') # for decoder -- attention right_word1 = Input(shape=(X_max_len, ), dtype='float32', name='input4') right_word2 = Input(shape=(X_max_len, ), dtype='float32', name='input5') right_word3 = Input(shape=(X_max_len, ), dtype='float32', name='input6') right_word4 = Input(shape=(X_max_len, ), dtype='float32', name='input7') left_word1 = Input(shape=(X_max_len, ), dtype='float32', name='input8') left_word2 = Input(shape=(X_max_len, ), dtype='float32', name='input9') left_word3 = Input(shape=(X_max_len, ), dtype='float32', name='input10') left_word4 = Input(shape=(X_max_len, ), dtype='float32', name='input11') phonetic_input = Input(shape=(n_phonetic_features, ), dtype='float32', name='input12') emb_layer1 = Embedding(X_vocab_len, EMBEDDING_DIM, input_length=X_max_len, mask_zero=False, name='Embedding') list_of_inputs = [ current_word, right_word1, right_word2, right_word3, right_word4, left_word1, left_word2, left_word3, left_word4 ] current_word_embedding, right_word_embedding1, right_word_embedding2,right_word_embedding3, right_word_embedding4, \ left_word_embedding1, left_word_embedding2, left_word_embedding3, left_word_embedding4 = [emb_layer1(i) for i in list_of_inputs] print("Type:: ", type(current_word_embedding)) list_of_embeddings1 = [current_word_embedding, right_word_embedding1, right_word_embedding2,right_word_embedding3, right_word_embedding4, \ left_word_embedding1, left_word_embedding2, left_word_embedding3, left_word_embedding4] list_of_embeddings = [ Dropout(0.50, name='drop1_' + str(j))(i) for i, j in zip(list_of_embeddings1, range(len(list_of_embeddings1))) ] list_of_embeddings = [ GaussianNoise(0.05, name='noise1_' + str(j))(i) for i, j in zip(list_of_embeddings, range(len(list_of_embeddings))) ] conv4_curr, conv4_right1, conv4_right2, conv4_right3, conv4_right4, conv4_left1, conv4_left2, conv4_left3, conv4_left4 =\ [Conv1D(filters=no_filters, kernel_size=4, padding='valid',activation='relu', strides=1, name='conv4_'+str(j))(i) for i,j in zip(list_of_embeddings, range(len(list_of_embeddings)))] conv4s = [ conv4_curr, conv4_right1, conv4_right2, conv4_right3, conv4_right4, conv4_left1, conv4_left2, conv4_left3, conv4_left4 ] maxPool4 = [ MaxPooling1D(name='max4_' + str(j))(i) for i, j in zip(conv4s, range(len(conv4s))) ] avgPool4 = [ AveragePooling1D(name='avg4_' + str(j))(i) for i, j in zip(conv4s, range(len(conv4s))) ] pool4_curr, pool4_right1, pool4_right2, pool4_right3, pool4_right4, pool4_left1, pool4_left2, pool4_left3, pool4_left4 = \ [merge([i,j], name='merge_conv4_'+str(k)) for i,j,k in zip(maxPool4, avgPool4, range(len(maxPool4)))] conv5_curr, conv5_right1, conv5_right2, conv5_right3, conv5_right4, conv5_left1, conv5_left2, conv5_left3, conv5_left4 = \ [Conv1D(filters=no_filters, kernel_size=5, padding='valid', activation='relu', strides=1, name='conv5_'+str(j))(i) for i,j in zip(list_of_embeddings, range(len(list_of_embeddings)))] conv5s = [ conv5_curr, conv5_right1, conv5_right2, conv5_right3, conv5_right4, conv5_left1, conv5_left2, conv5_left3, conv5_left4 ] maxPool5 = [ MaxPooling1D(name='max5_' + str(j))(i) for i, j in zip(conv5s, range(len(conv5s))) ] avgPool5 = [ AveragePooling1D(name='avg5_' + str(j))(i) for i, j in zip(conv5s, range(len(conv5s))) ] pool5_curr, pool5_right1, pool5_right2, pool5_right3, pool5_right4, pool5_left1, pool5_left2, pool5_left3, pool5_left4 = \ [merge([i,j], name='merge_conv5_'+str(k)) for i,j,k in zip(maxPool5, avgPool5, range(len(maxPool5)))] maxPools = [pool4_curr, pool4_right1, pool4_right2, pool4_right3, pool4_right4, \ pool4_left1, pool4_left2, pool4_left3, pool4_left4, \ pool5_curr, pool5_right1, pool5_right2, pool5_right3, pool5_right4, \ pool5_left1, pool5_left2, pool5_left3, pool5_left4] concat = merge(maxPools, mode='concat', name='main_merge') x = Dropout(0.15, name='drop_single1')(concat) x = Bidirectional(RNN(rnn_output_size), name='bidirec1')(x) total_features = [x, phonetic_input] concat2 = merge(total_features, mode='concat', name='phonetic_merging') x = Dense(HIDDEN_DIM, activation='relu', kernel_initializer='he_normal', kernel_constraint=maxnorm(3), bias_constraint=maxnorm(3), name='dense1')(concat2) x = Dropout(0.15, name='drop_single2')(x) x = Dense(HIDDEN_DIM, kernel_initializer='he_normal', activation='tanh', kernel_constraint=maxnorm(3), bias_constraint=maxnorm(3), name='dense2')(x) x = Dropout(0.15, name='drop_single3')(x) out1 = Dense(n1, kernel_initializer='he_normal', activation='softmax', name='output1')(x) out2 = Dense(n2, kernel_initializer='he_normal', activation='softmax', name='output2')(x) out3 = Dense(n3, kernel_initializer='he_normal', activation='softmax', name='output3')(x) out4 = Dense(n4, kernel_initializer='he_normal', activation='softmax', name='output4')(x) out5 = Dense(n5, kernel_initializer='he_normal', activation='softmax', name='output5')(x) out6 = Dense(n6, kernel_initializer='he_normal', activation='softmax', name='output6')(x) # Luong et al. 2015 attention model emb_layer = Embedding(X_vocab_len, EMBEDDING_DIM, input_length=X_max_len, mask_zero=True, name='Embedding_for_seq2seq') current_word_embedding, right_word_embedding1, right_word_embedding2,right_word_embedding3, right_word_embedding4, \ left_word_embedding1, left_word_embedding2, left_word_embedding3, left_word_embedding4 = [emb_layer(i) for i in list_of_inputs] # current_word_embedding = smart_merge([ current_word_embedding, right_word_embedding1, left_word_embedding1]) encoder, state = GRU(rnn_output_size, return_sequences=True, unroll=True, return_state=True, name='encoder')(current_word_embedding) encoder_last = encoder[:, -1, :] decoder = emb_layer(decoder_input) decoder = GRU(rnn_output_size, return_sequences=True, unroll=True, name='decoder')(decoder, initial_state=[encoder_last]) attention = dot([decoder, encoder], axes=[2, 2], name='dot') attention = Activation('softmax', name='attention')(attention) context = dot([attention, encoder], axes=[2, 1], name='dot2') decoder_combined_context = concatenate([context, decoder], name='concatenate') outputs = TimeDistributed(Dense(64, activation='tanh'), name='td1')(decoder_combined_context) outputs = TimeDistributed(Dense(X_vocab_len, activation='softmax'), name='td2')(outputs) all_inputs = [current_word, decoder_input, right_word1, right_word2, right_word3, right_word4, left_word1, left_word2, left_word3,\ left_word4, phonetic_input] all_outputs = [outputs, out1, out2, out3, out4, out5, out6] model = Model(input=all_inputs, output=all_outputs) opt = Adam() return model
def build_model(args): print("args", vars(args)) model = Sequential() np.random.seed(args.seed) if hasattr(args, 'embedding_weights') and args.embedding_weights is not None: W = np.load(args.embedding_weights) if args.train_embeddings: model.add(Embedding(args.n_vocab, args.n_word_dims, weights=[W], W_constraint=maxnorm(args.embedding_max_norm))) else: model.add(ImmutableEmbedding(args.n_vocab, args.n_word_dims, weights=[W])) else: model.add(Embedding(args.n_vocab, args.n_word_dims, mask_zero=args.mask_zero, W_constraint=maxnorm(args.embedding_max_norm))) model.add(LSTM(args.n_word_dims, args.n_units, truncate_gradient=args.truncate_gradient, return_sequences=True)) if args.regularization_layer == 'dropout': model.add(Dropout(0.2)) #elif args.regularization_layer == 'normalization': # model.add(BatchNormalization((args.n_filters,))) model.add(LSTM(args.n_units, args.n_units, truncate_gradient=args.truncate_gradient, return_sequences=True)) if args.regularization_layer == 'dropout': model.add(Dropout(0.2)) #elif args.regularization_layer == 'normalization': # model.add(BatchNormalization((args.n_filters,))) ''' model.add(LSTM(args.n_units, args.n_units, truncate_gradient=args.truncate_gradient, return_sequences=True)) if args.regularization_layer == 'dropout': model.add(Dropout(0.2)) #elif args.regularization_layer == 'normalization': # model.add(BatchNormalization((args.n_filters,))) ''' model.add(LSTM(args.n_units, args.n_units, truncate_gradient=args.truncate_gradient, return_sequences=False)) if args.regularization_layer == 'dropout': model.add(Dropout(0.2)) #elif args.regularization_layer == 'normalization': # model.add(BatchNormalization((args.n_filters,))) model.add(Dense(args.n_units, args.n_classes, W_regularizer=l2(args.l2_penalty))) model.add(Activation('softmax')) if args.optimizer == 'SGD': optimizer = SGD(lr=args.learning_rate, decay=args.decay, momentum=args.momentum, clipnorm=args.clipnorm) elif args.optimizer == 'Adam': optimizer = Adam(clipnorm=args.clipnorm) elif args.optimizer == 'RMSprop': optimizer = RMSprop(clipnorm=args.clipnorm) elif args.optimizer == 'Adadelta': optimizer = Adadelta(clipnorm=args.clipnorm) elif args.optimizer == 'Adagrad': optimizer = Adagrad(clipnorm=args.clipnorm) else: raise ValueError("don't know how to use optimizer {0}".format(args.optimizer)) model.compile(loss=args.loss, optimizer=optimizer) return model
seed = 7 numpy.random.seed(seed) # load data (X_train, y_train), (X_test, y_test) = cifar10.load_data() # normalize inputs from 0-255 to 0.0-1.0 X_train = X_train.astype('float32') X_test = X_test.astype('float32') X_train = X_train / 255.0 X_test = X_test / 255.0 # one hot encode outputs y_train = np_utils.to_categorical(y_train) y_test = np_utils.to_categorical(y_test) num_classes = y_test.shape[1] # Create the model model = Sequential() model.add(Conv2D(32, (3, 3), input_shape=(3, 32, 32), padding='same', activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.2)) model.add(Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(512, activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) # Compile model epochs = 25 lrate = 0.01 decay = lrate/epochs sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False) #tbCallBack= keras.callbacks.TensorBoard(log_dir='./Graph’, histogram_freq=0,write_graph=True, write_images=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
(X_train, y_train), (X_test, y_test) = cifar10.load_data() # normalize inputs from 0-255 to 0.0-1.0 X_train = X_train.astype('float32') X_test = X_test.astype('float32') X_train = X_train / 255.0 X_test = X_test / 255.0 # one hot encode outputs y_train = np_utils.to_categorical(y_train) y_test = np_utils.to_categorical(y_test) num_classes = y_test.shape[1] # Create the model model = Sequential() model.add(Convolution2D(32, 3, 3, input_shape=(3, 32, 32), border_mode='same', activation='relu', W_constraint=maxnorm(3))) model.add(Dropout(0.2)) model.add(Convolution2D(32, 3, 3, activation='relu', border_mode='same', W_constraint=maxnorm(3))) # plot_filters(model[0, ]) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(512, activation='relu', W_constraint=maxnorm(3))) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) # Compile model epochs = 1 learning_rate = 0.01 decay = learning_rate / epochs sgd = SGD(lr=learning_rate, momentum=0.9, decay=decay, nesterov=False)
def DNN_RUN(filename, Epochs=100, ROTATION=4, LAYER_NUM=2, Node_num=100, Train_Batch=50): ####################################### SELF SETTING !!!!! if (filename[0] == "/"): filename = filename elif (filename[0] == '~'): filename = filename.replace("~", os.environ['HOME']) elif ((filename[0] == "C") & (filename[1] == ":")): filename = filename else: filename = os.getcwd( ) + "/" + filename # get the path included filename loca = len(filename) for i in range(1, len(filename) + 1): # find the "/" location if (filename[-i] == "/"): loca = i - 1 break FILENAME = filename.replace(filename[:-loca], "") # this is the shorten filename filename_No_Txt = FILENAME.replace(".txt", "") infile = filename data = TFile.Open(infile) dirlist = data.GetListOfKeys() ITER = dirlist.MakeIterator() key = ITER.Next() TREE_NAME = (key.ReadObj()).GetName() #print(TREE_NAME) tree = data.Get(TREE_NAME) TOT_ENTRY = tree.GetEntries() UPPER_LIMIT = int(TOT_ENTRY * float(ROTATION - 1) / float(ROTATION)) upper_limit = UPPER_LIMIT # 1000 EPOCHS = Epochs #NODENUM = 128 NODENUM = Node_num BATCH_SIZE_train = Train_Batch # 500 BATCH_SIZE_test = 1 # 1 OPTIMIZER = 'adam' #'adam', 'Adamax', 'adagrad', 'sgd', 'adadelta', 'RMSprop', 'Nadam', 'TFOptimizer' KERNAL_INIT = 'he_uniform' #truncated_normal, glorot_uniform, random_uniform, he_uniform, he_normal L2 = 1e-5 #1e-5 DROP_RATE = 0.0 # 0.2 MAXNORM = 10000000 #5 ACTIVATION = "relu" ####################################### Input DATA Sets !!!!! b_reco_lep1Pt_ = tree2array(tree, branches="reco_lep1Pt") b_reco_lep1Eta_ = tree2array(tree, branches="reco_lep1Eta") b_reco_lep1Phi_ = tree2array(tree, branches="reco_lep1Phi") b_reco_lep2Pt_ = tree2array(tree, branches="reco_lep2Pt") b_reco_lep2Eta_ = tree2array(tree, branches="reco_lep2Eta") b_reco_lep2Phi_ = tree2array(tree, branches="reco_lep2Phi") b_reco_MET_et_ = tree2array(tree, branches="reco_MET_et") b_reco_MET_phi_ = tree2array(tree, branches="reco_MET_phi") ############################################################################################################### ##################################### Target DATA !!!!! b_TARGET_ = tree2array(tree, branches="gen_Nu_phi1") ############################################################################################################### ##################################### Pick valid Input/Target DATA (TARGET is output data) !!!!! b_reco_lep1Pt = np.zeros(b_reco_lep1Pt_.size) b_reco_lep1Eta = np.zeros(b_reco_lep1Eta_.size) b_reco_lep1Phi = np.zeros(b_reco_lep1Phi_.size) b_reco_lep2Pt = np.zeros(b_reco_lep2Pt_.size) b_reco_lep2Eta = np.zeros(b_reco_lep2Eta_.size) b_reco_lep2Phi = np.zeros(b_reco_lep2Phi_.size) b_reco_MET_et = np.zeros(b_reco_MET_et_.size) b_reco_MET_phi = np.zeros(b_reco_MET_phi_.size) b_TARGET = np.zeros(b_TARGET_.size) for i in range(b_TARGET.size): b_reco_lep1Pt[i] = b_reco_lep1Pt_[i] b_reco_lep1Eta[i] = b_reco_lep1Eta_[i] b_reco_lep1Phi[i] = b_reco_lep1Phi_[i] b_reco_lep2Pt[i] = b_reco_lep2Pt_[i] b_reco_lep2Eta[i] = b_reco_lep2Eta_[i] b_reco_lep2Phi[i] = b_reco_lep2Phi_[i] b_reco_MET_et[i] = b_reco_MET_et_[i] b_reco_MET_phi[i] = b_reco_MET_phi_[i] b_TARGET[i] = b_TARGET_[i] ############################################################################################################### ##################################### ARRAY is input DATA !!!!! ARRAY = np.stack( (b_reco_lep1Pt, b_reco_lep1Eta, b_reco_lep1Phi, b_reco_lep2Pt, b_reco_lep2Eta, b_reco_lep2Phi, b_reco_MET_et, b_reco_MET_phi)) print(ARRAY.shape) ARRAY = ARRAY.T print(ARRAY.shape) print(ARRAY[:2]) print(b_TARGET[:2]) ############################################################################################################### ndim = ARRAY.shape[1] nEvents = ARRAY.shape[0] rotation = 0 for i in range(ROTATION): # print("This is ",i+1,"th rotation, Of total", ROTATION) rotation = rotation + 1 if (rotation == 1): LOAD_WEIGHTS = False if (rotation != 1): LOAD_WEIGHTS = True if (rotation == 1): data_Temp_train = ARRAY[0:upper_limit] target_Temp_train = b_TARGET[0:upper_limit] data_Temp_test = ARRAY[(upper_limit + 1):ARRAY.shape[0]] target_Temp_test = b_TARGET[(upper_limit + 1):ARRAY.shape[0]] elif (rotation == ROTATION): data_Temp_train = ARRAY[int(TOT_ENTRY / float(ROTATION)) + 1:TOT_ENTRY] target_Temp_train = b_TARGET[int(TOT_ENTRY / float(ROTATION)) + 1:TOT_ENTRY] data_Temp_test = ARRAY[0:int(TOT_ENTRY / float(ROTATION))] target_Temp_test = b_TARGET[0:int(TOT_ENTRY / float(ROTATION))] else: INIT1 = 0 END1 = TOT_ENTRY - int(TOT_ENTRY / float(ROTATION)) * (rotation) INIT2 = TOT_ENTRY - int( TOT_ENTRY / float(ROTATION)) * (rotation - 1) END2 = TOT_ENTRY data_Temp_train1 = ARRAY[INIT1:END1] target_Temp_train1 = b_TARGET[INIT1:END1] data_Temp_train2 = ARRAY[INIT2 + 1:END2] target_Temp_train2 = b_TARGET[INIT2 + 1:END2] data_Temp_test = ARRAY[END1 + 1:INIT2] target_Temp_test = b_TARGET[END1 + 1:INIT2] data_Temp_train = np.concatenate( (data_Temp_train1, data_Temp_train2), axis=0) target_Temp_train = np.concatenate( (target_Temp_train1, target_Temp_train2), axis=0) data_train = data_Temp_train target_train = target_Temp_train data_test = data_Temp_test target_test = target_Temp_test ijkl = 0 modelRegress = Sequential() #modelRegress.add(Dropout(DROP_RATE, input_shape=(ndim,))) #modelRegress.add(Dense(NODENUM, kernel_initializer=KERNAL_INIT, activation=ACTIVATION, W_regularizer=l2(L2), kernel_constraint=maxnorm(3))) modelRegress.add( Dense(NODENUM, kernel_initializer=KERNAL_INIT, activation=ACTIVATION, W_regularizer=l2(L2), input_dim=ndim, kernel_constraint=maxnorm(MAXNORM))) #modelRegress.add(BatchNormalization()) #modelRegress.add(Activation(ACTIVATION)) modelRegress.add(Dropout(DROP_RATE)) ijkl = ijkl + 1 if (LAYER_NUM >= 2): modelRegress.add( Dense(NODENUM, kernel_initializer=KERNAL_INIT, activation=ACTIVATION, W_regularizer=l2(L2), kernel_constraint=maxnorm(MAXNORM))) modelRegress.add(Dropout(DROP_RATE)) ijkl = ijkl + 1 if (LAYER_NUM >= 3): modelRegress.add( Dense(NODENUM, kernel_initializer=KERNAL_INIT, activation=ACTIVATION, W_regularizer=l2(L2))) #additional layer modelRegress.add(Dropout(DROP_RATE)) ijkl = ijkl + 1 if (LAYER_NUM >= 4): modelRegress.add( Dense(NODENUM, kernel_initializer=KERNAL_INIT, activation=ACTIVATION, W_regularizer=l2(L2))) #additional layer modelRegress.add(Dropout(DROP_RATE)) ijkl = ijkl + 1 if (LAYER_NUM >= 5): modelRegress.add( Dense(NODENUM, kernel_initializer=KERNAL_INIT, activation=ACTIVATION, W_regularizer=l2(L2))) #additional layer modelRegress.add(Dropout(DROP_RATE)) ijkl = ijkl + 1 modelRegress.add( Dense(1, kernel_initializer=KERNAL_INIT, activation='linear')) #sigmoid modelRegress.compile(loss='mean_squared_error', optimizer=OPTIMIZER) if (LOAD_WEIGHTS == True): LOAD_MODEL_WEIGHTS = "MODELs/Model_EN" + str( upper_limit) + "_LN" + str(ijkl) + "_E" + str( EPOCHS) + "_M" + str(rotation - 1) + ".h5" modelRegress.load_weights(LOAD_MODEL_WEIGHTS, by_name=True) print( "************************************ MODEL LOADED ************************************" ) modelRegress.summary() print("************************************ This is ", i + 1, "th rotation, Of total", ROTATION, "************************************") history_callback = modelRegress.fit(data_train, target_train, validation_data=(data_test, target_test), epochs=EPOCHS, batch_size=BATCH_SIZE_train) predict_train = modelRegress.predict(data_train, batch_size=BATCH_SIZE_test) predict_test = modelRegress.predict(data_test, batch_size=BATCH_SIZE_test) modelRegress.save_weights("MODELs/Model_EN" + str(upper_limit) + "_LN" + str(ijkl) + "_E" + str(EPOCHS) + "_M" + str(rotation) + ".h5") if (rotation == ROTATION): f = TFile( "Rotation_tree_EN" + str(upper_limit) + "_LN" + str(ijkl) + "_E" + str(EPOCHS) + "_NN" + str(NODENUM) + "_B" + str(BATCH_SIZE_train) + "_" + str(OPTIMIZER) + "_L" + str(L2) + "_DR" + str(DROP_RATE) + ".root", "recreate") hist_target_train = TH1F('TrainData', 'TrainData', 20, -3.5, 3.5) hist_target_test = TH1F('TestData', 'TestData', 20, -3.5, 3.5) hist_output_train = TH1F('OutputDataTrain', 'OutputDataTrain', 20, -3.5, 3.5) hist_output_test = TH1F('OutputDataTest', 'OutputDataTest', 20, -3.5, 3.5) tree_train = TTree('tree_train', 'tree_train') Ttrain = np.zeros(1, dtype=float) Otrain = np.zeros(1, dtype=float) tree_train.Branch('target_train', Ttrain, 'target_train/D') tree_train.Branch('output_train', Otrain, 'output_train/D') for ij1 in range(upper_limit - 2): Ttrain[0] = target_train[ij1] Otrain[0] = predict_train[ij1] tree_train.Fill() tree_test = TTree('tree_test', 'tree_test') Ttest = np.zeros(1, dtype=float) Otest = np.zeros(1, dtype=float) tree_test.Branch('target_test', Ttest, 'target_test/D') tree_test.Branch('output_test', Otest, 'output_test/D') for ij2 in range(ARRAY.shape[0] - upper_limit - 3): Ttest[0] = target_test[ij2] Otest[0] = predict_test[ij2] tree_test.Fill() fill_hist(hist_target_train, target_train) fill_hist(hist_target_test, target_test) fill_hist(hist_output_train, predict_train[:, 0]) fill_hist(hist_output_test, predict_test[:, 0]) f.Write() f.Close() CP_FNAME = "MODELs/Model_EN" + str(upper_limit) + "_LN" + str( ijkl) + "_E" + str(EPOCHS) + "_M" + str(ROTATION) + ".h5" CP_TNAME = "MODELs/FINAL_EN" + str(upper_limit) + "_LN" + str( ijkl) + "_EP" + str(EPOCHS) + "_M" + str(ROTATION) + ".h5" CP_FINAL_MODEL = "cp " + CP_FNAME + " " + CP_TNAME RM_COMMAND = "rm " RM_NAMES = "MODELs/Model_EN" + str(upper_limit) + "_LN" + str( ijkl) + "_E" + str(EPOCHS) + "_M" + "*" + ".h5" RM_COMMAND = RM_COMMAND + RM_NAMES os.system(CP_FINAL_MODEL) os.system(RM_COMMAND)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) print('Build model...') input_sentence = Input(shape=(maxlen,), dtype='int32') # we start off with an efficient embedding layer which maps # our vocab indices into embedding_dims dimensions input_embedding = Embedding(max_features, embedding_dims, weights=[np.load('mr_word2vec_300_dim_google.embeddings')])(input_sentence) cnns = [Convolution1D(filter_length=filter_length, nb_filter=nb_filter, W_regularizer=l2(0.0001), W_constraint=maxnorm(3), activation='relu', border_mode='same') for filter_length in [2, 3, 5, 7]] cnn_feature = merge([cnn(input_embedding) for cnn in cnns], mode='concat') dropout = Dropout(0.25) sentence_dropout = dropout(cnn_feature) maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) sentence_pool = maxpool(sentence_dropout) predict_sentiment = Dense(2, activation='softmax')(sentence_pool) model = Model(input=[input_sentence], output=[predict_sentiment])
def get_model(self,opt): text_branch = Sequential() embedding_layer = Embedding(len(opt.word_index) + 1,opt.embedding_dim,weights=[opt.embedding_matrix],input_length=opt.max_sequence_length,trainable=False) text_branch.add(embedding_layer) # text_branch.add(TrigPosEmbedding( # input_shape=(None,), # output_dim=EMBEDDING_DIM, # The dimension of embeddings. # mode=TrigPosEmbedding.MODE_ADD, # Use `add` mode; MODE_CONCAT # name='Pos-Embd', # )) # text_branch.add(Bidirectional(LSTM(units=100,return_sequences=False))) text_branch.add(self.rnncell(units=self.opt.hidden_unit_num,return_sequences=True, activation='relu' ,kernel_constraint=maxnorm(3))) text_branch.add(Dropout(self.opt.dropout_rate)) text_branch.add(self.rnncell(units=self.opt.hidden_unit_num,return_sequences=False, activation='relu' ,kernel_constraint=maxnorm(3))) text_branch.add(Dropout(self.opt.dropout_rate)) # text_branch.add(SeqSelfAttention(attention_width=5,attention_activation='sigmoid')) text_branch.add(Dense(3,activation="softmax")) return text_branch
activation='relu', border_mode='same')) model.add(Dropout(0.2)) model.add(Convolution2D(32, 3, 3, activation='relu', border_mode='same')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Convolution2D(64, 3, 3, activation='relu', border_mode='same')) model.add(Dropout(0.2)) model.add(Convolution2D(64, 3, 3, activation='relu', border_mode='same')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Convolution2D(128, 3, 3, activation='relu', border_mode='same')) model.add(Dropout(0.2)) model.add(Convolution2D(128, 3, 3, activation='relu', border_mode='same')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dropout(0.2)) model.add(Dense(1024, activation='relu', W_constraint=maxnorm(3))) model.add(Dropout(0.2)) model.add(Dense(512, activation='relu', W_constraint=maxnorm(3))) model.add(Dropout(0.2)) model.add(Dense(num_classes, activation='softmax')) # Compile model epochs = 25 lrate = 0.01 decay = lrate / epochs sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) print(model.summary())
def train(save_best=True): physical_devices = tf.config.list_physical_devices('GPU') try: tf.config.experimental.set_memory_growth(physical_devices[0], True) except: # Invalid device or cannot modify virtual devices once initialized. pass print("Training CNN classifier...") # fix random seed for reproducibility seed = 7 np.random.seed(seed) # TODO: data_path = "test" names = os.listdir(data_path) input_res = (72, 128) # load data from folders based on folder name: images = [] labels = [] label_dict = {} num_images = 0 for i, name in enumerate(names): # count examples image_files = os.listdir(data_path + '/' + name) num_images += len(image_files) im_path = data_path + '/' + names[0] # encoded category for name label_dict.update({name: i}) # load image into example pool for im_name in image_files: im = data_path + '/' + name + '/' + im_name im = cv.imread(im) image = cv.resize(im, dsize=(input_res)) images.append(image) labels.append(i) # Normalize pixel values to be between 0 and 1 images = np.asarray(images) images = images / 255.0 # encode labels train_images, test_images, train_labels, test_labels = train_test_split(images, labels, test_size=0.33) # Create the model model = Sequential() model.add(Conv2D(16, kernel_size=3, padding='same', activation='relu', input_shape=(input_res[0], input_res[1], 3))) model.add(Dropout(0.1)) model.add(MaxPooling2D((2, 2))) model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu')) model.add(Dropout(0.1)) model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu')) model.add(Dropout(0.1)) model.add(MaxPooling2D((2, 2))) model.add(Conv2D(64, kernel_size=3, padding='same', activation='relu')) model.add(Dropout(0.2)) model.add(MaxPooling2D((2, 2))) model.add(Conv2D(64, kernel_size=5, padding='same', activation='relu')) model.add(MaxPooling2D((2, 2))) model.add(Flatten()) model.add(Dropout(0.4)) model.add(Dense(2048, activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.5)) model.add(Dense(2048, activation='relu', kernel_constraint=maxnorm(3))) model.add(Dropout(0.5)) model.add(Dense(10)) # Compile model model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy']) model.summary() callbacks_list = [] if save_best: filepath = "best_facial_classifier.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max') callbacks_list.append(checkpoint) history = model.fit(train_images, train_labels, batch_size=64, epochs=500, validation_data=(test_images, test_labels), callbacks=callbacks_list) return [model, history]
import cv2 from PIL import Image # fix random seed for reproducibility seed = 7 numpy.random.seed(seed) mode = 'v' num_classes = 3 output = ['Sad','Happy','Angry'] # Create the model model = Sequential() model.add(Convolution2D(32, 3, 3, input_shape=(1, 32, 32), border_mode='same', activation='relu', W_constraint=maxnorm(3))) model.add(Dropout(0.2)) model.add(Convolution2D(32, 3, 3, activation='relu', border_mode='same', W_constraint=maxnorm(3))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(512, activation='relu', W_constraint=maxnorm(3))) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) model.load_weights("models/model-0.h5") # Compile model epochs = 25
def def_model(self, model_id=0): input_shape = self.input_shape if model_id == 0: # vgg 16 input_tensor = Input(shape=input_shape) x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(input_tensor) x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x) # Block 2 x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x) x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x) # Block 3 x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x) x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x) x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x) # Block 4 x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x) # Classification block x = Flatten(name='flatten')(x) x = Dense(4096, activation='relu', name='fc1')(x) x = Dense(4096, activation='relu', name='fc2')(x) x = Dropout(0.5)(x) x = Dense(10, activation='softmax', name='predictions')(x) return_model = Model(inputs=[input_tensor], outputs=[x]) return_model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(), metrics=['accuracy']) elif model_id == 1: """ Convolutional Neural Network: https://github.com/umbertogriffo/Fashion-mnist-cnn-keras/blob/ master/src/convolutional/fashion_mnist_cnn.py """ return_model = Sequential() return_model.add( Conv2D(32, (5, 5), input_shape=self.input_shape, padding='same', activation='relu')) return_model.add(MaxPooling2D(pool_size=(2, 2), padding='same')) return_model.add( Conv2D(64, (5, 5), padding='same', activation='relu')) return_model.add(MaxPooling2D(pool_size=(2, 2), padding='same')) return_model.add( Conv2D(128, (1, 1), padding='same', activation='relu')) return_model.add(MaxPooling2D(pool_size=(2, 2), padding='same')) return_model.add(Flatten()) return_model.add( Dense(1024, activation='relu', kernel_constraint=maxnorm(3))) return_model.add(Dropout(0.5)) return_model.add( Dense(512, activation='relu', kernel_constraint=maxnorm(3))) return_model.add(Dropout(0.5)) return_model.add(Dense(self.num_classes, activation='softmax')) # Compile model lrate = 0.1 decay = lrate / self.epoch sgd = keras.optimizers.SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=True) return_model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) elif model_id == 2: # https://github.com/cmasch/zalando-fashion-mnist/blob/master/Simple_Convolutional_Neural_Network_Fashion-MNIST.ipynb cnn = Sequential() cnn.add(InputLayer(input_shape=self.input_shape)) cnn.add(BatchNormalization()) cnn.add( Convolution2D(64, (4, 4), padding='same', activation='relu')) cnn.add(MaxPooling2D(pool_size=(2, 2))) cnn.add(Dropout(0.1)) cnn.add(Convolution2D(64, (4, 4), activation='relu')) cnn.add(MaxPooling2D(pool_size=(2, 2))) cnn.add(Dropout(0.3)) cnn.add(Flatten()) cnn.add(Dense(256, activation='relu')) cnn.add(Dropout(0.5)) cnn.add(Dense(64, activation='relu')) cnn.add(BatchNormalization()) cnn.add(Dense(self.num_classes, activation='softmax')) cnn.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adam(), metrics=['accuracy']) return cnn elif model_id == 3: pass # https://github.com/markjay4k/Fashion-MNIST-with-Keras/blob/master/pt%204%20-%20Deeper%20CNNs.ipynb else: raise Exception("unsupported model") return return_model