def __init__(self, output_dimesion, vocab_size, dropout_rate, emb_dim, max_len, nb_filters, init_W=None, cae_N_hidden=50, nb_features=17): ''' CNN Module''' model_summary = open('cnn_cae_transfer_model_summary', 'w') self.max_len = max_len max_features = vocab_size vanila_dimension = 200 projection_dimension = output_dimesion filter_lengths = [3, 4, 5] # self.model = Graph() # input doc_input = Input(shape=(max_len, ), dtype='int32', name='doc_input') '''Embedding Layer''' if init_W is None: # self.model.add_node(Embedding( # max_features, emb_dim, input_length=max_len), name='sentence_embeddings', input='input') sentence_embeddings = Embedding( output_dim=emb_dim, input_dim=max_features, input_length=max_len, name='sentence_embeddings')(doc_input) else: # self.model.add_node(Embedding(max_features, emb_dim, input_length=max_len, weights=[ # init_W / 20]), name='sentence_embeddings', input='input') sentence_embeddings = Embedding( output_dim=emb_dim, input_dim=max_features, input_length=max_len, weights=[init_W / 20], name='sentence_embeddings')(doc_input) '''Reshape Layer''' reshape = Reshape(target_shape=(max_len, emb_dim, 1), name='reshape')(sentence_embeddings) # chanels last '''Convolution Layer & Max Pooling Layer''' flatten_ = [] for i in filter_lengths: model_internal = Sequential() # model_internal.add(Conv2D( # nb_filters, i, emb_dim, activation="relu")) model_internal.add( Conv2D(nb_filters, (i, emb_dim), activation="relu", name='conv2d_' + str(i), input_shape=(self.max_len, emb_dim, 1))) # model_internal.add(MaxPooling2D( # pool_size=(self.max_len - i + 1, 1))) model_internal.add( MaxPooling2D(pool_size=(self.max_len - i + 1, 1), name='maxpool2d_' + str(i))) model_internal.add(Flatten()) # plot_model(model_internal, 'model_cnn_cae_transfer_conv2d_%d.png'%i, show_shapes=True) model_internal.summary( print_fn=lambda x: model_summary.write(x + '\n')) flatten = model_internal(reshape) flatten_.append(flatten) ''' Attributes module ''' lam = 1e-3 N = nb_features # cae_N_hidden = 50 att_input = Input(shape=(N, ), name='cae_input') encoded = Dense(cae_N_hidden, activation='tanh', name='encoded')(att_input) att_output = Dense(N, activation='linear', name='cae_output')(encoded) # model = Model(input=att_input, output=att_output) def contractive_loss(y_pred, y_true): mse = K.mean(K.square(y_true - y_pred), axis=1) W = K.variable(value=model.get_layer('encoded').get_weights() [0]) # N x cae_N_hidden W = K.transpose(W) # cae_N_hidden x N h = model.get_layer('encoded').output dh = h * (1 - h) # N_batch x cae_N_hidden # N_batch x cae_N_hidden * cae_N_hidden x 1 = N_batch x 1 contractive = lam * K.sum(dh**2 * K.sum(W**2, axis=1), axis=1) return mse + contractive '''Transfer layer ''' # if cae_N_hidden != nb_filters: # sys.exit("For the transfer layer to work ''for now'' the attributes latent vector dimension (--att_dim)" # " must equal the number of filters (kernal) of the conv. layer (--num_kernel_per_ws)") # combine the outputs of boths modules model_internal = Sequential(name='Transfer_ResBlock') model_internal.add( Conv1D(cae_N_hidden / 2, 1, activation="relu", name='Res_conv2d_1', input_shape=(cae_N_hidden, 1))) model_internal.add( Conv1D(cae_N_hidden, 1, activation="relu", name='Res_conv2d_2')) model_internal.add( MaxPooling1D(pool_size=cae_N_hidden, name='Res_maxpool1d')) model_internal.add(Flatten()) reshape = Reshape(target_shape=(cae_N_hidden, 1), name='reshape_encoded')(encoded) # chanels last residual = model_internal(reshape) transfered = residual # plot_model(model_internal,'model_cnn_cae_transfer_transferblock.png',show_shapes=True) model_internal.summary( print_fn=lambda x: model_summary.write(x + '\n')) # shortcut = encoded # transfered = add([residual, shortcut]) ''' Adding CAE output to CNN output ''' flatten_.append(transfered) '''Fully Connect Layer & Dropout Layer''' # self.model.add_node(Dense(vanila_dimension, activation='tanh'), # name='fully_connect', inputs=['unit_' + str(i) for i in filter_lengths]) fully_connect = Dense(vanila_dimension, activation='tanh', name='fully_connect')(concatenate(flatten_, axis=-1)) # self.model.add_node(Dropout(dropout_rate), # name='dropout', input='fully_connect') dropout = Dropout(dropout_rate, name='dropout')(fully_connect) # joint_output = add([dropout, transfered], name='joint_output') '''Projection Layer & Output Layer''' # self.model.add_node(Dense(projection_dimension, activation='tanh'), # name='projection', input='dropout') pj = Dense(projection_dimension, activation='tanh', name='joint_output') # output layer projection = pj(dropout) # Output Layergit model = Model(inputs=[doc_input, att_input], outputs=[projection, att_output]) # todo: check the optimizer model.compile( optimizer='rmsprop', # optimizer={'joint_output': 'rmsprop', 'cae_output': 'adam'}, loss={ 'joint_output': 'mse', 'cae_output': contractive_loss }, loss_weights={ 'joint_output': 1., 'cae_output': 1. }) # plot_model(model, to_file='model.png') self.model = model # plot_model(model, to_file='model_cnn_cae_transfer.png',show_shapes=True) model.summary(print_fn=lambda x: model_summary.write(x + '\n'))
def build_model(): ''' Build a Keras model with the functional API ''' bn_mode = 1 if args.chars: char_input = Input(shape=(args.max_word_len, ), dtype='int32', name='char_input') x = Embedding(char_vocab_size, args.char_embedding_dim, input_length=args.max_word_len)(char_input) x = Reshape((args.max_word_len, args.char_embedding_dim))(x) prev_x = x for rnet_idx in range(args.resnet): if args.bn: x = BatchNormalization()(x) x = Activation('relu')(x) if args.dropout: x = Dropout(args.dropout)(x) x = Convolution1D(args.char_embedding_dim, 8, activation='relu', padding='same')(x) if args.bn: x = BatchNormalization()(x) x = Activation('relu')(x) if args.dropout: x = Dropout(args.dropout)(x) x = Convolution1D(args.char_embedding_dim, 4, activation='relu', padding='same')(x) x = add([prev_x, x]) x = MaxPooling1D(pool_size=2, padding='same')(x) prev_x = x if args.bn: x = BatchNormalization()(x) x = Activation('linear')(x) feature_size = args.char_embedding_dim char_embedding = Reshape( (int(args.max_word_len / (2**args.resnet)), int(feature_size)))(x) if args.words: word_input = Input(shape=(args.max_sent_len, ), dtype='int32', name='word_input') if not args.ignore_embeddings: word_embedding = Embedding(vocab_size, word_embedding_dim, input_length=args.max_sent_len, weights=[embedding_weights], trainable=(not args.freeze), name='word_embedding')(word_input) else: word_embedding = Embedding(vocab_size, word_embedding_dim, input_length=args.max_sent_len, name='word_embedding')(word_input) l = GRU(units=int(args.rnn_dim), return_sequences=False, dropout=args.dropout, input_shape=(args.max_sent_len, word_embedding_dim), activation='relu')(word_embedding) #l = GRU(units=int(args.rnn_dim)/2, return_sequences=False, dropout=args.dropout, input_shape=(args.max_sent_len, word_embedding_dim), activation='relu')(l) r = GRU(units=int(args.rnn_dim), return_sequences=False, go_backwards=True, dropout=args.dropout, input_shape=(args.max_sent_len, word_embedding_dim), activation='relu')(word_embedding) #r = GRU(units=int(args.rnn_dim)/2, return_sequences=False, go_backwards=True, dropout=args.dropout, input_shape=(args.max_sent_len, word_embedding_dim), activation='relu')(r) word_x = concatenate([l, r]) if args.bn: word_x = BatchNormalization()(word_x) if args.dropout: word_x = Dropout(args.dropout)(word_x) if args.chars: embedding = char_embedding if args.bn: embedding = BatchNormalization()(embedding) if args.rnn: # Bidirectional GRU l = GRU(units=int(args.rnn_dim), return_sequences=False, dropout=args.dropout, input_shape=(args.max_sent_len, args.char_embedding_dim), activation='relu')(embedding) #l = GRU(units=int(args.rnn_dim)/2, return_sequences=False, dropout=args.dropout, input_shape=(args.max_sent_len, args.char_embedding_dim), activation='relu')(l) r = GRU(units=int(args.rnn_dim), return_sequences=False, go_backwards=True, dropout=args.dropout, input_shape=(args.max_sent_len, args.char_embedding_dim), activation='relu')(embedding) #r = GRU(units=int(args.rnn_dim)/2, return_sequences=False, go_backwards=True, dropout=args.dropout, input_shape=(args.max_sent_len, args.char_embedding_dim), activation='relu')(r) x = concatenate([l, r]) if args.bn: x = BatchNormalization()(x) else: x = Convolution1D(args.char_embedding_dim, 8, activation='relu', border_mode='same')(embedding) # x = MaxPooling1D(pool_length=8, border_mode='same')(x) x = Convolution1D(args.char_embedding_dim, 4, activation='relu', border_mode='same')(x) # x = MaxPooling1D(pool_length=8, border_mode='same')(x) x = Reshape((args.char_embedding_dim * 2, ))(x) x = Dense(args.char_embedding_dim, activation='relu')(x) if args.chars and args.words: x = concatenate([x, word_x]) x = Dense(word_embedding_dim * 2, activation='relu')(x) elif args.words: x = word_x anger_output = Dense(1, activation='sigmoid', name='anger_output')(x) anticipation_output = Dense(1, activation='sigmoid', name='anticipation_output')(x) disgust_output = Dense(1, activation='sigmoid', name='disgust_output')(x) fear_output = Dense(1, activation='sigmoid', name='fear_output')(x) joy_output = Dense(1, activation='sigmoid', name='joy_output')(x) love_output = Dense(1, activation='sigmoid', name='love_output')(x) optimism_output = Dense(1, activation='sigmoid', name='optimism_output')(x) pessimism_output = Dense(1, activation='sigmoid', name='pessimism_output')(x) sadness_output = Dense(1, activation='sigmoid', name='sadness_output')(x) surprise_output = Dense(1, activation='sigmoid', name='surprise_output')(x) trust_output = Dense(1, activation='sigmoid', name='trust_output')(x) main_output = Dense(1, activation='linear', name='main_output')(x) if args.chars and args.words: model_input = [word_input, char_input] elif args.chars: model_input = [ char_input, ] elif args.words: model_input = [ word_input, ] model_output = [ main_output, anger_output, anticipation_output, disgust_output, fear_output, joy_output, love_output, optimism_output, pessimism_output, sadness_output, surprise_output, trust_output ] model = Model(inputs=model_input, outputs=model_output) return model
y_test=np.delete(y_test,k) X_test_add=np.delete(X_test_add,k,0) print('Pad sequences (samples x time)') X_train = sequence.pad_sequences(X_train_one_hot, maxlen=maxlen) X_test = sequence.pad_sequences(X_test_one_hot, maxlen=maxlen) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) print('Build model...') #embedding model emb_model = Sequential() emb_model.add(Embedding(max_features, 128, input_length=maxlen,dropout=0.2)) emb_model.add(Convolution1D(nb_filter=32, filter_length=3, border_mode='same', activation='relu')) emb_model.add(MaxPooling1D(pool_length=2)) # ? may merge a layer here which include features from POST (grammar info) emb_model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2)) # try using a GRU instead, for fun #additional model add_model = Sequential() add_model.add(Dense(12, input_dim=2, init='normal', activation='relu',W_constraint = maxnorm(2))) #merge model model = Sequential() model.add(Merge([add_model, emb_model], mode='concat', concat_axis=1)) model.add(Dense(32)) model.add(Dense(1)) model.add(Activation('sigmoid'))
X_test = sequence.pad_sequences(X_test, maxlen=maxlen) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) print('Build model...') model = Sequential() model.add(Embedding(max_features, embedding_size, input_length=maxlen)) model.add(Dropout(0.25)) model.add( Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='valid', activation='relu', subsample_length=1)) model.add(MaxPooling1D(pool_length=pool_length)) model.add(LSTM(lstm_output_size)) model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) print('Train...') model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, validation_data=(X_test, y_test)) score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)
# CNN # =============================================================================================== print ("Method = CNN for Arabic Sentiment Analysis'") model_variation = 'CNN-non-static' np.random.seed(0) nb_filter = embeddings_dim main_input = Input(shape=(max_sent_len,)) embedding = Embedding(max_features, embeddings_dim, input_length=max_sent_len, mask_zero=False, weights=[embedding_weights] )(main_input) Drop1 = Dropout(dropout_prob[0])(embedding) convs = [] for n_gram in filter_sizes: conv = Convolution1D(nb_filter=nb_filter, filter_length=n_gram, border_mode='valid', activation='relu', subsample_length=1, input_dim=embeddings_dim, input_length=max_sent_len)(Drop1) pool = MaxPooling1D(pool_length=max_sent_len - n_gram + 1)(conv) # flat = Flatten()(pool) convs.append(pool) if len(filter_sizes) > 1: merged = merge(convs, mode='concat') else: merged = convs[0] conv_model = Model(inputs=main_input, outputs=merged) # add created model grapg in sequential model model = Sequential() model.add(conv_model) # add model just like layer model.add(Convolution1D(nb_filter=nb_filter, filter_length=filter_sizes[1], border_mode='same', activation='relu')) model.add(MaxPooling1D(pool_length=3, border_mode='same')) model.add(Dropout(0.25))
for filter_len in cfg.get('cnn', 'filtlen').split(','): branch = Sequential() branch.add( Embedding(len(dataset.word2int), cfg.getint('cnn', 'embdims'), input_length=maxlen, weights=init_vectors)) branch.add( Convolution1D(nb_filter=cfg.getint('cnn', 'filters'), filter_length=int(filter_len), border_mode='valid', activation='relu', subsample_length=1)) branch.add(MaxPooling1D(pool_length=2)) branch.add(Flatten()) branches.append(branch) train_xs.append(train_x) test_xs.append(test_x) model = Sequential() model.add(Merge(branches, mode='concat')) model.add(Dense(cfg.getint('cnn', 'hidden'))) model.add(Dropout(cfg.getfloat('cnn', 'dropout'))) model.add(Activation('relu')) model.add(Dropout(cfg.getfloat('cnn', 'dropout'))) model.add(Dense(classes))
def m11(num_classes=10): print('Using Model M11') m = Sequential() m.add( Conv1D(64, input_shape=[AUDIO_LENGTH, 1], kernel_size=80, strides=4, padding='same', kernel_initializer='glorot_uniform', kernel_regularizer=regularizers.l2(l=0.0001))) m.add(BatchNormalization()) m.add(Activation('relu')) m.add(MaxPooling1D(pool_size=4, strides=None)) for i in range(2): m.add( Conv1D(64, kernel_size=3, strides=1, padding='same', kernel_initializer='glorot_uniform', kernel_regularizer=regularizers.l2(l=0.0001))) m.add(BatchNormalization()) m.add(Activation('relu')) m.add(MaxPooling1D(pool_size=4, strides=None)) for i in range(2): m.add( Conv1D(128, kernel_size=3, strides=1, padding='same', kernel_initializer='glorot_uniform', kernel_regularizer=regularizers.l2(l=0.0001))) m.add(BatchNormalization()) m.add(Activation('relu')) m.add(MaxPooling1D(pool_size=4, strides=None)) for i in range(3): m.add( Conv1D(256, kernel_size=3, strides=1, padding='same', kernel_initializer='glorot_uniform', kernel_regularizer=regularizers.l2(l=0.0001))) m.add(BatchNormalization()) m.add(Activation('relu')) m.add(MaxPooling1D(pool_size=4, strides=None)) for i in range(2): m.add( Conv1D(512, kernel_size=3, strides=1, padding='same', kernel_initializer='glorot_uniform', kernel_regularizer=regularizers.l2(l=0.0001))) m.add(BatchNormalization()) m.add(Activation('relu')) m.add(Lambda(lambda x: K.mean(x, axis=1))) m.add(Dense(num_classes, activation='softmax')) return m
def RNN(X_train, y_train, args): """ Purpose -> Define and train the proposed LSTM network Input -> Data, Labels and model hyperparameters Output -> Trained LSTM network """ # Sets the model hyperparameters # Embedding hyperparameters max_features = args[0] maxlen = args[1] embedding_size = args[2] # Convolution hyperparameters filter_length = args[3] nb_filter = args[4] pool_length = args[5] # LSTM hyperparameters lstm_output_size = args[6] # Training hyperparameters batch_size = args[7] nb_epoch = args[8] numclasses = args[9] test_size = args[10] # Format conversion for y_train for compatibility with Keras y_train = np_utils.to_categorical(y_train, numclasses) print(y_train) # Train & Validation data splitting X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=test_size, random_state=42) # Build the sequential model # Model Architecture is: # Input -> Embedding -> Conv1D+Maxpool1D -> LSTM -> LSTM -> FC-1 -> Softmaxloss print('Build model...') start = time() log_dir = datetime.now().strftime('model_%Y%m%d_%H%M') os.mkdir(log_dir) es = EarlyStopping(monitor='val_loss', patience=20) mc = ModelCheckpoint(log_dir + '\\CIFAR10-EP{epoch:02d}-ACC{val_acc:.4f}.h5', monitor='val_loss', save_best_only=True) tb = TensorBoard(log_dir=log_dir, histogram_freq=0) sequence = Input(shape=(maxlen, ), dtype='int32') # embedded = Embedding(input_dim=max_features, output_dim=num_features, input_length=maxlen, mask_zero=True, weights=[W], trainable=False) (sequence) embedded = Embedding(input_dim=max_features, output_dim=embedding_size, input_length=maxlen, trainable=False)(sequence) embedded = Dropout(0.25)(embedded) convolution = Convolution1D(filters=nb_filter, filter_length=filter_length, padding='valid', activation='relu', strides=1)(embedded) maxpooling = MaxPooling1D(pool_length=pool_length)(convolution) lstm = LSTM(lstm_output_size, dropout_W=0.2, dropout_U=0.2, return_sequences=True)(maxpooling) lstm1 = LSTM(lstm_output_size, dropout_W=0.2, dropout_U=0.2, return_sequences=False)(lstm) enc = Bidirectional( GRU(lstm_output_size // 2, recurrent_dropout=0.25, return_sequences=True))(maxpooling) att = AttentionM()(enc) x = keras.layers.Concatenate(axis=1)([lstm1, att]) fc1 = Dense(128, activation="relu")(x) fc2 = Dense(64, activation="relu")(fc1) fc3 = Dense(32, activation="relu")(fc2) fc4 = Dense(16, activation="relu")(fc3) fc4_dropout = Dropout(0.25)(fc4) output = Dense(3, activation='softmax')(fc4_dropout) model = Model(inputs=sequence, outputs=output) '''model = Sequential() model.add(Embedding(max_features, embedding_size, input_length=maxlen)) model.add(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='valid', activation='relu', subsample_length=1)) model.add(MaxPooling1D(pool_length=pool_length)) model.add(SpatialDropout1D(0.1)) model.add(Bidirectional(CuDNNGRU(64, return_sequences=True))) model.add(Bidirectional(CuDNNGRU(64, return_sequences=True))) Routings = 5 Num_capsule = 10 Dim_capsule = 32 model.add(Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings, share_weights=True)) model.add(Flatten()) model.add(LSTM(lstm_output_size, dropout_W=0.2, dropout_U=0.2, return_sequences=True)) model.add(LSTM(lstm_output_size, dropout_W=0.2, dropout_U=0.2, return_sequences=True)) model.add(Bidirectional(LSTM(lstm_output_size//2, recurrent_dropout=0.25, return_sequences=False))) #model.add(AttentionM()) model.add(Dropout(0.25)) model.add(Dense(numclasses,activation='softmax'))''' # Optimizer is Adamax along with categorical crossentropy loss model.compile( loss='categorical_crossentropy', optimizer='adamax', metrics=['accuracy'], ) print(model.summary()) history = LossHistory() print('Train...') # Trains model for 50 epochs with shuffling after every epoch for training data and validates on validation data model.fit(X_train, y_train, batch_size=batch_size, shuffle=True, nb_epoch=nb_epoch, validation_data=(X_valid, y_valid), callbacks=[history, es, mc, tb]) history.loss_plot('epoch') return model
# load model sorted_pi_list = np.load('/mnt/sorted_pi_list.out') model = load_model('best_model.h5') print("Loaded model") print(model.summary()) # make model that ends at last neuron pre-softmax c1 = model.get_layer("conv1d_1") b1 = model.get_layer("batch_normalization_1") d1 = model.get_layer("dense_1") b2 = model.get_layer("batch_normalization_2") d2 = model.get_layer("dense_2") model2 = Sequential() model2.add(Convolution1D(input_shape=(16048,4), filters=128, kernel_size=12, activation="relu", padding="same", weights=c1.get_weights())) model2.add(MaxPooling1D(pool_size=16048)) model2.add(BatchNormalization(weights=b1.get_weights())) model2.add(Flatten()) model2.add(Dense(input_dim=128,units=64, weights=d1.get_weights())) model2.add(Activation("relu")) model2.add(BatchNormalization(weights=b2.get_weights())) model2.add(Dense(units=827, weights=d2.get_weights())) model2.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=["accuracy"]) # cycle through each plasmid of interest for h in range(len(dna_seq_of_interest)): seq = dna_seq_of_interest[h] # intialize variables for plasmid disruption scan total_plasmid_size = len(seq)+disruption_len probability_for_correct = np.zeros(total_plasmid_size)
def main(args): if len(args) < 1: sys.stderr.write("Error - one required argument: <data directory>\n") sys.exit(-1) working_dir = args[0] data_file = os.path.join(working_dir, 'training-data.liblinear') # learn alphabet from training data provider = dataset.DatasetProvider(data_file) # now load training examples and labels train_x, train_y = provider.load(data_file) # turn x and y into numpy array among other things maxlen = max([len(seq) for seq in train_x]) classes = len(set(train_y)) train_x = pad_sequences(train_x, maxlen=maxlen) train_y = to_categorical(np.array(train_y), classes) #loading pre-trained embedding file: embeddings_index = {} f = open(os.path.join(working_dir, 'mimic.txt')) values = f.readline().split() EMBEDDING_WORDNUM = int(values[0]) EMBEDDING_DIM = int(values[1]) for line in f: values = line.split() word = values[0] coefs = np.asarray(values[1:], dtype='float32') embeddings_index[word] = coefs f.close() print('load embeddings for %s=%s words.' % (len(embeddings_index), EMBEDDING_WORDNUM)) # prepare embedding matrix nb_words = len(provider.word2int) embedding_matrix = np.zeros((nb_words, EMBEDDING_DIM)) for word, i in provider.word2int.items(): embedding_vector = embeddings_index.get(word) if embedding_vector is not None: # words not found in embedding index will be all-zeros. embedding_matrix[i] = embedding_vector pickle.dump(maxlen, open(os.path.join(working_dir, 'maxlen.p'), "wb")) pickle.dump(provider.word2int, open(os.path.join(working_dir, 'word2int.p'), "wb")) pickle.dump(provider.label2int, open(os.path.join(working_dir, 'label2int.p'), "wb")) print 'train_x shape:', train_x.shape print 'train_y shape:', train_y.shape branches = [] # models to be merged train_xs = [] # train x for each branch inflows = [] # placeholder for each input for filter_len in '2,5'.split(','): branch = Input(shape=(maxlen, )) embed = Embedding(len(provider.word2int), EMBEDDING_DIM, weights=[embedding_matrix], trainable=True)(branch) conv = Conv1D(filters=200, kernel_size=int(filter_len), padding='valid', activation='relu', strides=1)(embed) pool = MaxPooling1D(pool_size=2)(conv) flat = Flatten()(pool) branches.append(flat) train_xs.append(train_x) inflows.append(branch) concat = concatenate(branches) drop1 = Dropout(0.25)(concat) dense = Dense(200, activation='relu')(drop1) drop2 = Dropout(0.25)(dense) out = Dense(classes, activation='softmax')(drop2) model = Model(inputs=inflows, outputs=out) #optimizer = RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08) optimizer = Adam(lr=0.001) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) stopper = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='auto') model.fit(train_xs, train_y, epochs=20, batch_size=128, verbose=1, validation_split=0.1, callbacks=[stopper]) json_string = model.to_json() open(os.path.join(working_dir, 'model_0.json'), 'w').write(json_string) model.save_weights(os.path.join(working_dir, 'model_0.h5'), overwrite=True) sys.exit(0)
def lstm_model(): # model model = Sequential() # add conv1D layer model.add( Conv1D(filters=32, kernel_size=7, padding='same', dilation_rate=2, activation='relu', input_shape=(features_num, vec_size))) #model.add(Dropout(0.2)) model.add(MaxPooling1D(pool_size=3)) model.add(Dropout(0.2)) for i in range(4): # add conv1D layer model.add( Conv1D(filters=32, kernel_size=7, dilation_rate=2, padding='same', activation='relu')) model.add(MaxPooling1D(pool_size=3)) model.add(Dropout(0.2)) for i in range(5): model.add( Conv1D(filters=64, kernel_size=5, padding='same', activation='relu')) model.add(MaxPooling1D(pool_size=3)) model.add(Dropout(0.2)) for i in range(5): model.add( Conv1D(filters=128, kernel_size=3, padding='same', activation='relu')) model.add(MaxPooling1D(pool_size=3)) model.add(Dropout(0.2)) for i in range(5): model.add( Conv1D(filters=256, kernel_size=3, padding='same', activation='relu')) model.add(MaxPooling1D(pool_size=3)) model.add(Dropout(0.2)) # add LSTM layer model.add(LSTM(600, dropout=0.2, recurrent_dropout=0.2)) # model.add(Dropout(0.2)) model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam') return model
verbose, epochs, batch_size = 0, 50, 150 n_timesteps, n_features, n_outputs = n_timesteps, X_train.shape[2], y_train.shape[0] # reshape data into time steps of sub-sequences n_steps, n_length = 5, 50 X_train = X_train1.reshape((X_train1.shape[0], n_steps, n_length, n_features)) X_test = X_test1.reshape((X_test1.shape[0], n_steps, n_length, n_features)) # Initialising the model classifier = Sequential() #Adding the first layer of CNN classifier.add(TimeDistributed(Conv1D(filters=45, kernel_size=3, activation='relu'), input_shape=(None,n_length,n_features))) #Adding the second layer of CNN and some Dropout regularisation classifier.add(TimeDistributed(Conv1D(filters=45, kernel_size=3, activation='relu'))) classifier.add(TimeDistributed(Dropout(0.5))) #Adding a layer of MaxPooling and Flatten regularisation classifier.add(TimeDistributed(MaxPooling1D(pool_size=2))) classifier.add(TimeDistributed(Flatten())) # Adding the first LSTM layer and some Dropout regularisation classifier.add(LSTM(units = 100, return_sequences = True)) classifier.add(Dropout(0.5)) # Adding a second LSTM layer and some Dropout regularisation classifier.add(LSTM(units = 100, return_sequences = True)) classifier.add(Dropout(0.5)) # Adding a third LSTM layer and some Dropout regularisation classifier.add(LSTM(units = 100, return_sequences = True)) classifier.add(Dropout(0.5)) # Adding a fourth LSTM layer and some Dropout regularisation classifier.add(LSTM(units = 100)) classifier.add(Dropout(0.5)) # Adding the output layer classifier.add(Dense(1))
def distancenet(self, vocab_size, output_size, maxsize=1, hop_depth=-1, dropout=False, d_perc=1, type="CCE", shape=0, q_shape=1): print(bcolors.UNDERLINE + 'Building nn model...' + bcolors.ENDC) print(q_shape, shape, "====================") sentrnn = Sequential() emb = Embedding(vocab_size, self.embed_hidden_size, mask_zero=False, W_constraint=mx(), W_regularizer=reg(), init=init_function, input_shape=shape) sentrnn.add(emb) #emb_bn = bn() #sentrnn.add(emb_bn) sentrnn.add(Dropout(0.2)) if (convolution): #print maxsize, q_shape #exit() conv = Convolution1D(self.sent_hidden_size, maxsize, subsample_length=1, activation=act, border_mode="same") #conv_bn = bn() sentrnn.add(conv) #sentrnn.add(conv_bn) #sentrnn.add(Dropout(0.1)) sentrnn.add(MaxPooling1D(pool_length=maxsize)) #sentrnn.add(Dropout(d_perc)) else: sentrnn.add(MaxPooling1D(pool_length=maxsize)) #sentrnn.add(SimpleRNN( self.query_hidden_size, return_sequences=True,activation = "leakyrelu", init = init_function,dropout_W = 0.3, dropout_U=0.3, consume_less = "mem")) # sentrnn.add(ResettingRNN(maxsize, self.query_hidden_size, return_sequences=True,activation = "tanh", init = init_function,dropout_W = 0.3, dropout_U=0.3, consume_less = "mem")) # sentrnn.add(bn()) # sentrnn.add(AttentionPooling(pool_length=maxsize)) #sentrnn.add(Convolution1D(self.sent_hidden_size,maxsize, subsample_length = 1, activation=act, border_mode="same")) #sentrnn.add(bn()) # #sentrnn.add(Dropout(0.1)) #td = TimeDistributedDense(self.sent_hidden_size, activation = act, init = init_function) #sentrnn.add(td) #sentrnn.add(bn()) #sentrnn.add(Dropout(d_perc)) qrnn = Sequential() #emb = Embedding(vocab_size, self.embed_hidden_size, mask_zero=False,W_constraint=mx(), W_regularizer=reg(), init = init_function, input_shape=shape, input_length=q_shape) qrnn.add(InputLayer(input_shape=(q_shape, ))) qrnn.add(emb) #qrnn.add(emb_bn) qrnn.add(Dropout(0.2)) if (convolution): #conv = Convolution1D(self.sent_hidden_size,q_shape, subsample_length = 1, activation=act, border_mode="same") qrnn.add(conv) #qrnn.add(conv_bn) #qrnn.add(Dropout(0.1)) qrnn.add(MaxPooling1D(pool_length=q_shape)) qrnn.add(Flatten(1)) else: qrnn.add( SimpleRNN(self.query_hidden_size, return_sequences=False, activation=act, init=init_function, dropout_W=0.1, dropout_U=0.1, consume_less="mem")) #qrnn.add(Convolution1D(self.sent_hidden_size,q_shape, subsample_length = 1, activation=act, border_mode="same")) #qrnn.add(bn()) #qrnn.add(MaxPooling1D(pool_length=q_shape)) #qrnn.add(Flatten()) init_qa = [sentrnn, qrnn] past = [] #at = GRU(self.sent_hidden_size, dropout_W = 0.3, dropout_U=0.3, activation=act) td = TimeDistributedDense(self.sent_hidden_size, activation=act, init=init_function) at = AttentionRecurrent(self.sent_hidden_size, activation="softmax") for i in range(hop_depth): hop = Sequential() hop.add( AttentionMerge((i == hop_depth - 1), "abs", init_qa + past, mode="sum")) hop.add(td) #hop.add(bn()) #hop.add(Dropout(0.1)) hop.add(at) #hop.add(bn()) #hop.add(Dropout(0.1)) past.append(hop) # model = Sequential() # model.add(AttentionMerge(False, "concat", init_qa + past, mode = "concat")) # model.add(LSTM(self.query_hidden_size, return_sequences=False, init = init_function,dropout_W = 0.2, dropout_U=0.2)) model = hop #self._adddepth(model, dropout, d_perc) model.add( Dense(vocab_size, W_constraint=mx(), W_regularizer=reg(), init=init_function)) #model.add(bn()) model.add(Activation("softmax")) if (type == "CCE"): model.compile(optimizer=self._getopt(), loss='categorical_crossentropy', metrics=["accuracy"], class_mode='categorical') else: model.compile(optimizer=self._getopt(), loss='mse') print(model.summary()) plot(model, show_layer_names=False, show_shapes=True, to_file='model.png') return model
build_tweet.append(X_vecs[token]) vector_list.append(build_tweet) X_test = np.array(vector_list) # Keras CNN model del X_vecs batch_size = 64 nb_epochs = 50 model = Sequential() model.add(Conv1D(64, kernel_size=3, activation='elu', padding='same', input_shape=(max_tweet_length, vector_size))) model.add(Conv1D(128, kernel_size=3, activation='elu', padding='same')) model.add(Dropout(0.25)) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(Conv1D(256, kernel_size=3, activation='elu', padding='same')) model.add(Conv1D(512, kernel_size=3, activation='elu', padding='same')) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(256, activation='tanh')) model.add(Dropout(0.5)) model.add(Dense(1, activation='sigmoid')) # Compile the model model.compile( loss='binary_crossentropy',
def build_model(top_words, embedding_vecor_length, max_review_length, show_summaries=False): input_layer = Embedding(top_words, embedding_vecor_length, input_length=max_review_length) # --- 3 --- branch_3 = Sequential() branch_3.add(input_layer) branch_3.add( Conv1D(filters=128, kernel_size=3, padding='same', kernel_regularizer=l2(.01))) branch_3.add(Activation('relu')) branch_3.add(MaxPooling1D(pool_size=2)) branch_3.add(Dropout(0.5)) branch_3.add(BatchNormalization()) branch_3.add(LSTM(128)) # --- 5 --- branch_5 = Sequential() branch_5.add(input_layer) branch_5.add( Conv1D(filters=128, kernel_size=5, padding='same', kernel_regularizer=l2(.01))) branch_5.add(Activation('relu')) branch_5.add(MaxPooling1D(pool_size=2)) branch_5.add(Dropout(0.5)) branch_5.add(BatchNormalization()) branch_5.add(LSTM(128)) # --- 7 --- branch_7 = Sequential() branch_7.add(input_layer) branch_7.add( Conv1D(filters=128, kernel_size=7, padding='same', kernel_regularizer=l2(.01))) branch_7.add(Activation('relu')) branch_7.add(MaxPooling1D(pool_size=2)) branch_7.add(Dropout(0.5)) branch_7.add(BatchNormalization()) branch_7.add(LSTM(128)) # --- 9 --- branch_9 = Sequential() branch_9.add(input_layer) branch_9.add( Conv1D(filters=128, kernel_size=9, padding='same', kernel_regularizer=l2(.01))) branch_9.add(Activation('relu')) branch_9.add(MaxPooling1D(pool_size=2)) branch_9.add(Dropout(0.5)) branch_9.add(BatchNormalization()) branch_9.add(LSTM(128)) model = Sequential() model.add(Merge([branch_3, branch_5, branch_7, branch_9], mode='concat')) model.add(Dense(1, activation='sigmoid')) opt = keras.optimizers.SGD() model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy']) if show_summaries: print(branch_3.summary()) print(branch_4.summary()) print(branch_5.summary()) print(model.summary()) return model
print("BUILDING MODEL") embedding_vecor_length = 32 input_layer = Embedding(top_words, embedding_vecor_length, input_length=max_review_length) branch_3 = Sequential() branch_3.add(input_layer) branch_3.add( Conv1D(filters=32, kernel_size=3, padding='same', kernel_regularizer=l2(.01))) branch_3.add(Activation('relu')) branch_3.add(MaxPooling1D(pool_size=2)) branch_3.add(Dropout(0.5)) branch_3.add(BatchNormalization()) branch_3.add(LSTM(100)) branch_4 = Sequential() branch_4.add(input_layer) branch_4.add( Conv1D(filters=32, kernel_size=4, padding='same', kernel_regularizer=l2(.01))) branch_4.add(Activation('relu')) branch_4.add(MaxPooling1D(pool_size=2)) branch_4.add(Dropout(0.5)) branch_4.add(BatchNormalization())
n_steps_in, n_steps_out = 14, 7 X, y = split_sequence(data, n_steps_in, n_steps_out) # X.shape is [samples, timesteps] # reshape from [samples, timesteps] into [samples, subsequences, timesteps, features] n_features = 1 n_seq = 2 n_steps = 7 X = X.reshape((X.shape[0], n_seq, n_steps, n_features)) # define CNN-LSTM model model = Sequential() model.add( TimeDistributed(Conv1D(64, 1, activation='relu'), input_shape=(None, n_steps, n_features))) model.add(TimeDistributed(MaxPooling1D())) model.add(TimeDistributed(Flatten())) model.add(LSTM(50, activation='relu')) model.add(Dense(n_steps_out)) model.compile(optimizer='adam', loss='mse', metrics=['accuracy']) model.fit(X, y, epochs=200, verbose=1) X_test = np.array(data[-n_steps_in:]).reshape(1, n_seq, n_steps, n_features) y_hat = model.predict(X_test, verbose=1) print(f'y = {y[-n_steps_out:]}, y_hat = {y_hat}') print(model.summary)
def run(train_file, test_file, batch, epochs, embdims, filters, filtlen, hidden, dropout, learnrt): """Train/test with given parameters. Return F1.""" np.random.seed(1337) print 'train:', train_file print 'test:', test_file print 'batch:', batch print 'epochs:', epochs print 'embdims:', embdims print 'filters:', filters print 'filtlen:', filtlen print 'hidden:', hidden print 'dropout:', dropout print 'learnrt:', learnrt # learn alphabet from training examples datset = dataset.DatasetProvider(train_file) # now load training examples and labels train_x, train_y = datset.load(train_file) maxlen = max([len(seq) for seq in train_x]) # now load test examples and labels test_x, test_y = datset.load(test_file, maxlen=maxlen) # turn x and y into numpy array among other things classes = len(set(train_y)) train_x = pad_sequences(train_x, maxlen=maxlen) train_y = to_categorical(np.array(train_y), classes) test_x = pad_sequences(test_x, maxlen=maxlen) test_y = to_categorical(np.array(test_y), classes) branches = [] # models to be merged train_xs = [] # train x for each branch test_xs = [] # test x for each branch for filter_len in filtlen.split(','): branch = Sequential() branch.add( Embedding(len(datset.word2int), embdims, trainable=False, input_length=maxlen)) branch.add( Convolution1D(nb_filter=filters, filter_length=int(filter_len), border_mode='valid', activation='relu', subsample_length=1)) branch.add(MaxPooling1D(pool_length=2)) branch.add(Flatten()) branches.append(branch) train_xs.append(train_x) test_xs.append(test_x) model = Sequential() model.add(Merge(branches, mode='concat')) model.add(Dropout(dropout)) model.add(Dense(hidden)) model.add(Activation('relu')) model.add(Dropout(dropout)) model.add(Dense(classes)) model.add(Activation('softmax')) optimizer = RMSprop(lr=learnrt, rho=0.9, epsilon=1e-08) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) model.fit(train_xs, train_y, nb_epoch=epochs, batch_size=batch, verbose=0, validation_split=0.0, class_weight=None) # probability for each class; (test size, num of classes) distribution = \ model.predict(test_xs, batch_size=batch) # class predictions; (test size,) predictions = np.argmax(distribution, axis=1) # gold labels; (test size,) gold = np.argmax(test_y, axis=1) # f1 scores label_f1 = f1_score(gold, predictions, average=None) print for label, idx in datset.label2int.items(): print 'f1(%s)=%f' % (label, label_f1[idx]) if 'contains' in datset.label2int: idxs = [datset.label2int['contains'], datset.label2int['contains-1']] contains_f1 = f1_score(gold, predictions, labels=idxs, average='micro') print '\nf1(contains average) =', contains_f1 else: idxs = datset.label2int.values() average_f1 = f1_score(gold, predictions, labels=idxs, average='micro') print 'f1(all) =', average_f1 print '******************************************'
from keras.datasets import imdb from keras.models import Sequential from keras.layers import Dense from keras.layers import Flatten from keras.layers.convolutional import Conv1D from keras.layers.convolutional import MaxPooling1D from keras.layers.embeddings import Embedding from keras.preprocessing import sequence # load the dataset but only keep the top n words, zero the rest top_words = 5000 (X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=top_words) # pad dataset to a maximum review length in words max_words = 500 X_train = sequence.pad_sequences(X_train, maxlen=max_words) X_test = sequence.pad_sequences(X_test, maxlen=max_words) # create the model model = Sequential() model.add(Embedding(top_words, 32, input_length=max_words)) model.add(Conv1D(64, 3, padding='same', activation='relu')) model.add(MaxPooling1D()) model.add(Flatten()) model.add(Dense(250, activation='relu')) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() # Fit the model model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=2, batch_size=128, verbose=2) # Final evaluation of the model scores = model.evaluate(X_test, y_test, verbose=0) print("Accuracy: %.2f%%" % (scores[1]*100))
# model.compile(loss='mean_squared_error', optimizer='adam') # cnnblstm train_xinitial = train_xinitial.reshape(train_xinitial.shape[0], 2, int( train_xinitial.shape[1] / 2), train_xinitial.shape[2]) train_x = train_x.reshape(train_x.shape[0], 2, int( train_x.shape[1] / 2), train_x.shape[2]) test_x = test_x.reshape(test_x.shape[0], 2, int( test_x.shape[1] / 2), test_x.shape[2]) val_x = val_x.reshape(val_x.shape[0], 2, int( val_x.shape[1] / 2), val_x.shape[2]) model = Sequential() model.add(TimeDistributed(Conv1D(filters=128, kernel_size=1, activation='tanh'), input_shape=(None, train_x.shape[2], train_x.shape[3]))) model.add(TimeDistributed(MaxPooling1D(pool_size=4))) # # #print(model.summary()) # model.add(TimeDistributed(Conv1D(filters=64, kernel_size=1, activation='tanh'), input_shape=(None, train_x.shape[2],train_x.shape[3]))) # model.add(TimeDistributed(MaxPooling1D(pool_size=2))) # # #print(model.summary()) # model.add(TimeDistributed(Conv1D(filters=64, kernel_size=2, activation='tanh'), input_shape=(None, train_x.shape[2],train_x.shape[3]))) # model.add(TimeDistributed(MaxPooling1D(pool_size=4))) # model.add(TimeDistributed(Conv1D(filters=32, kernel_size=2, activation='tanh'), input_shape=(None, train_x.shape[2],train_x.shape[3]))) # model.add(TimeDistributed(MaxPooling1D(pool_size=4))) model.add(TimeDistributed(Flatten())) # print(model.summary()) model.add(Bidirectional(LSTM(60, return_sequences=True))) model.add(Bidirectional(LSTM(48, return_sequences=True))) model.add(Bidirectional(LSTM(48))) model.add(Dropout(0.5))
def make_model(num_tfs, num_bws, num_motifs, num_recurrent, num_dense, dropout_rate): from keras import backend as K from keras.models import Model from keras.layers import Dense, Dropout, Activation, Flatten, Layer, merge, Input from keras.layers.convolutional import Convolution1D, MaxPooling1D from keras.layers.pooling import GlobalMaxPooling1D from keras.layers.recurrent import LSTM from keras.layers.wrappers import Bidirectional, TimeDistributed ''' import tensorflow as tf config = tf.ConfigProto(device_count={'gpu':1}) config.gpu_options.allow_growth=True session = tf.Session(config=config) ''' forward_input = Input(shape=(L, 4 + num_bws,)) reverse_input = Input(shape=(L, 4 + num_bws,)) if num_recurrent < 0: hidden_layers = [ Convolution1D(input_dim=4 + num_bws, nb_filter=num_motifs, filter_length=w, border_mode='valid', activation='relu', subsample_length=1), Dropout(0.1), TimeDistributed(Dense(num_motifs, activation='relu')), GlobalMaxPooling1D(), Dropout(dropout_rate), Dense(num_dense, activation='relu'), Dropout(dropout_rate), Dense(num_tfs, activation='sigmoid') ] elif num_recurrent == 0: hidden_layers = [ Convolution1D(input_dim=4 + num_bws, nb_filter=num_motifs, filter_length=w, border_mode='valid', activation='relu', subsample_length=1), Dropout(0.1), TimeDistributed(Dense(num_motifs, activation='relu')), MaxPooling1D(pool_length=w2, stride=w2), Dropout(dropout_rate), Flatten(), Dense(num_dense, activation='relu'), Dropout(dropout_rate), Dense(num_tfs, activation='sigmoid') ] else: hidden_layers = [ Convolution1D(input_dim=4 + num_bws, nb_filter=num_motifs, filter_length=w, border_mode='valid', activation='relu', subsample_length=1), Dropout(0.1), TimeDistributed(Dense(num_motifs, activation='relu')), MaxPooling1D(pool_length=w2, stride=w2), Bidirectional(LSTM(num_recurrent, dropout_W=0.1, dropout_U=0.1, return_sequences=True)), Dropout(dropout_rate), Flatten(), Dense(num_dense, activation='relu'), Dropout(dropout_rate), Dense(num_tfs, activation='sigmoid') ] forward_output = get_output(forward_input, hidden_layers) reverse_output = get_output(reverse_input, hidden_layers) output = merge([forward_output, reverse_output], mode='ave') model = Model(input=[forward_input, reverse_input], output=output) return model
print(X_test.shape[0], 'test samples') # convert class vectors to binary class matrices Y_train = np_utils.to_categorical(y_train.astype(int), nb_classes) Y_test = np_utils.to_categorical(y_test.astype(int), nb_classes) print('Y_train shape:', Y_train.shape) model = Sequential() model.add( Convolution1D(nb_filters1, nb_conv, border_mode='valid', input_shape=(time, freq))) model.add(Activation('relu')) model.add(MaxPooling1D(pool_length=4)) model.add(Dropout(0.75)) model.add(Convolution1D( nb_filters1, nb_conv, border_mode='valid', )) model.add(Activation('relu')) model.add(MaxPooling1D(pool_length=2)) #model.add(Dropout(0.25)) model.add(Convolution1D( nb_filters2, nb_conv, border_mode='valid',
def cnn_lstm(train_x, test_x, train_y, test_y): if not os.path.exists("models"): os.mkdir("models") class_weights = \ class_weight.compute_class_weight('balanced', np.unique(train_y), train_y) class_weights = dict(enumerate(class_weights)) scaler = StandardScaler() scaler.fit(train_x) train_x = scaler.transform(train_x) test_x = scaler.transform(test_x) train_y = to_categorical(train_y) test_y = to_categorical(test_y) # define model verbose, epochs, batch_size = 1, 25, 32 print(train_x.shape) print(train_y.shape) n_timesteps, n_outputs = train_x.shape[1], train_y.shape[1] n_steps = 10 n_features = 1 # reshape data into time steps of sub-sequences n_length = int(n_timesteps / n_steps) train_x = train_x.reshape( (train_x.shape[0], n_steps, n_length, n_features)) test_x = test_x.reshape((test_x.shape[0], n_steps, n_length, n_features)) print(train_x.shape) print(test_x.shape) # define model model = Sequential() model.add( TimeDistributed(Conv1D(filters=64, kernel_size=3, activation='relu'), input_shape=(None, n_length, n_features))) model.add( TimeDistributed(Conv1D(filters=64, kernel_size=3, activation='relu'))) model.add(TimeDistributed(Dropout(0.5))) model.add(TimeDistributed(MaxPooling1D(pool_size=2))) model.add(TimeDistributed(Flatten())) model.add(LSTM(80)) model.add(Dropout(0.5)) model.add(Dense(30, activation='relu')) model.add(Dense(n_outputs, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # fit network checkpoint = \ ModelCheckpoint("models/physiological_cnn_model.h5", monitor='val_accuracy', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1) early_stopping = \ EarlyStopping(monitor='val_loss', patience=50, verbose=1, mode='auto') model.summary() model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, verbose=verbose, class_weight=class_weights, validation_data=(np.array(test_x), np.array(test_y)), callbacks=[checkpoint, early_stopping]) # evaluate model _, accuracy = model.evaluate(test_x, test_y, batch_size=batch_size, verbose=0) physiological_model = load_model("models/physiological_cnn_model.h5") preds_physiological = physiological_model.predict_proba(np.array(test_x)) print(accuracy) return preds_physiological
####################################### # build model ####################################### print 'building model' model = Sequential() model.add( Convolution1D(input_dim=4, input_length=maxlen, nb_filter=nb_filter, filter_length=filter_length, border_mode="valid", activation="relu", subsample_length=1)) model.add(MaxPooling1D(pool_length=13, stride=13)) #model.add(Dropout(0.2)) ''' model.add(brnn) model.add(Dropout(0.5)) model.add(Flatten()) model.add(Dense(input_dim=75*640, output_dim=925)) model.add(Activation('relu')) model.add(Dense(input_dim=925, output_dim=919)) model.add(Activation('sigmoid')) '''
def build_reference_annotation_1d_model_from_args(args, conv_width = 6, conv_layers = [128, 128, 128, 128], conv_dropout = 0.0, conv_batch_normalize = False, spatial_dropout = True, max_pools = [], padding='valid', activation = 'relu', annotation_units = 16, annotation_shortcut = False, annotation_batch_normalize = True, fc_layers = [64], fc_dropout = 0.0, fc_batch_normalize = False, fc_initializer = 'glorot_normal', kernel_initializer = 'glorot_normal', alpha_dropout = False ): '''Build Reference 1d CNN model for classifying variants. Architecture specified by parameters. Dynamically sets input channels based on args via defines.total_input_channels_from_args(args) Uses the functional API. Prints out model summary. Arguments args.annotations: The variant annotations, perhaps from a HaplotypeCaller VCF. args.labels: The output labels (e.g. SNP, NOT_SNP, INDEL, NOT_INDEL) Returns The keras model ''' in_channels = tensor_maps.total_input_channels_from_args(args) concat_axis = -1 x = reference = Input(shape=(args.window_size, in_channels), name=args.tensor_name) max_pool_diff = len(conv_layers)-len(max_pools) for i,c in enumerate(conv_layers): if conv_batch_normalize: x = Conv1D(filters=c, kernel_size=conv_width, activation='linear', padding=padding, kernel_initializer=kernel_initializer)(x) x = BatchNormalization(axis=concat_axis)(x) x = Activation(activation)(x) else: x = Conv1D(filters=c, kernel_size=conv_width, activation=activation, padding=padding, kernel_initializer=kernel_initializer)(x) if conv_dropout > 0 and alpha_dropout: x = AlphaDropout(conv_dropout)(x) elif conv_dropout > 0 and spatial_dropout: x = SpatialDropout1D(conv_dropout)(x) elif conv_dropout > 0: x = Dropout(conv_dropout)(x) if i >= max_pool_diff: x = MaxPooling1D(max_pools[i-max_pool_diff])(x) f = Flatten()(x) annotations = annotations_in = Input(shape=(len(args.annotations),), name=args.annotation_set) if annotation_batch_normalize: annotations_in = BatchNormalization(axis=concat_axis)(annotations_in) annotation_mlp = Dense(units=annotation_units, kernel_initializer=fc_initializer, activation=activation)(annotations_in) x = layers.concatenate([f, annotation_mlp], axis=1) for fc in fc_layers: if fc_batch_normalize: x = Dense(units=fc, activation='linear', kernel_initializer=fc_initializer)(x) x = BatchNormalization(axis=1)(x) x = Activation(activation)(x) else: x = Dense(units=fc, activation=activation, kernel_initializer=fc_initializer)(x) if fc_dropout > 0 and alpha_dropout: x = AlphaDropout(fc_dropout)(x) elif fc_dropout > 0: x = Dropout(fc_dropout)(x) if annotation_shortcut: x = layers.concatenate([x, annotations_in], axis=1) prob_output = Dense(units=len(args.labels), activation='softmax', name='softmax_predictions')(x) model = Model(inputs=[reference, annotations], outputs=[prob_output]) adam = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=1.) model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=get_metrics(args.labels)) model.summary() if os.path.exists(args.weights_hd5): model.load_weights(args.weights_hd5, by_name=True) print('Loaded model weights from:', args.weights_hd5) return model
def run(model_name, dataset_name): """ run the baseline model :param model_name: name of baseline models, CNN or LSTM :param dataset_name: name of datasets, candidate values [bbc, digg, MySpace, rw, Twitter, YouTube] """ print "model: %s" % model_name print "process dataset %s..." % dataset_name data = cPickle.load(open('./pkl/%s.pkl' % dataset_name, 'rb')) records, glove_embeddings, vocab, word_to_df = data dim_emb = len(glove_embeddings[1]) # index of word starts from 1 # initial weights of embedding layer embeddings = np.zeros((len(vocab) + 1, dim_emb), dtype='float32') for w in vocab: wid = vocab[w] embeddings[wid, :] = glove_embeddings[wid] train_x, train_y, val_x, val_y, test_x, test_y, test_strength = [], [], [], [], [], [], [] max_len = 0 for r in records: text = r['text'] y = r['label'] wids = [vocab[w] for w in text.split(' ')] if len(wids) > max_len: max_len = len(wids) if r['type'] == 'train': train_x.append(wids) train_y.append(y) elif r['type'] == 'val': val_x.append(wids) val_y.append(y) elif r['type'] == 'test': strength = r['strength'] test_x.append(wids) test_y.append(y) test_strength.append(strength) train_x, val_x, test_x = ToArray(train=train_x, val=val_x, test=test_x) train_y, val_y, test_y = ToArray(train=train_y, val=val_y, test=test_y) _, _, test_strength = ToArray(train=[], val=[], test=test_strength) #print train_x.shape, val_x.shape, test_x.shape train_x, val_x, test_x = Padding(train=train_x, val=val_x, test=test_x, max_len=max_len) batch_size = 50 if model_name == 'CNN' else 32 if train_x.shape[0] % batch_size: n_extra = batch_size - train_x.shape[0] % batch_size x_extra = train_x[:n_extra, :] y_extra = train_y[:n_extra] train_x = np.append(train_x, x_extra, axis=0) train_y = np.append(train_y, y_extra, axis=0) np.random.seed(38438) # shuffle the training set train_set = np.random.permutation(zip(train_x, train_y)) train_x, train_y = [], [] for (x, y) in train_set: train_x.append(x) train_y.append(y) n_labels = 2 train_x = np.array(train_x) train_y = np.array(train_y) train_y = to_categorical(train_y) val_y = to_categorical(val_y) print "n_train: %s, n_val: %s, n_test: %s" % ( train_x.shape[0], val_x.shape[0], test_x.shape[0]) if model_name == 'CNN': model = Graph() model.add_input(name='input', input_shape=(max_len, ), dtype='int') model.add_node(Embedding(input_dim=len(vocab) + 1, output_dim=dim_emb, input_length=max_len, weights=[embeddings]), name="emb", input="input") filter_hs = [3, 4, 5] n_filter = 100 dropout_rate = 0.5 n_epoch = 20 for i in xrange(len(filter_hs)): win_size = filter_hs[i] conv_name = 'conv%s' % i pool_name = "pool%s" % i flatten_name = "flatten%s" % i pool_size = max_len - win_size + 1 model.add_node( layer=Convolution1D(nb_filter=n_filter, filter_length=win_size, activation='relu', W_constraint=maxnorm(m=3), b_constraint=maxnorm(m=3)), name=conv_name, input='emb', ) model.add_node(layer=MaxPooling1D(pool_length=pool_size), name=pool_name, input=conv_name) model.add_node(layer=Flatten(), name=flatten_name, input=pool_name) model.add_node(layer=Dropout(p=dropout_rate), name="dropout", inputs=["flatten0", "flatten1", "flatten2"]) model.add_node(layer=Dense(output_dim=n_labels, activation='softmax'), name='softmax', input='dropout') model.add_output(input='softmax', name="output") model.compile(loss={'output': 'categorical_crossentropy'}, optimizer='adadelta', metrics=['accuracy']) model_path = './model/%s_%s.hdf5' % (model_name, dataset_name) best_model = ModelCheckpoint(filepath=model_path, monitor='val_acc', save_best_only=True, mode='max') print "training..." model.fit(data={ 'input': train_x, 'output': train_y }, batch_size=batch_size, nb_epoch=n_epoch, validation_data={ 'input': val_x, 'output': val_y }, callbacks=[best_model], verbose=0) else: model = Sequential() model.add( Embedding(input_dim=len(vocab) + 1, output_dim=dim_emb, mask_zero=True, input_length=max_len, weights=[embeddings])) model.add(LSTM(output_dim=128, dropout_W=0.2, dropout_U=0.2)) model.add(Dense(n_labels, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model_path = './model/%s_%s.hdf5' % (model_name, dataset_name) best_model = ModelCheckpoint(filepath=model_path, monitor='val_acc', save_best_only=True, mode='max') n_epoch = 20 print "training..." model.fit(x=train_x, y=train_y, batch_size=batch_size, nb_epoch=n_epoch, validation_data=(val_x, val_y), callbacks=[best_model], verbose=0) pred_strength = [] print "load the best model from disk..." model.load_weights(model_path) if model_name == 'LSTM': pred_strength = model.predict(x=test_x, batch_size=batch_size) else: for i in xrange(len(test_x)): res = model.predict(data={'input': test_x[i:i + 1]}, batch_size=1) pred_strength.append(res['output']) pred_strength = np.array(pred_strength) pred_strength = pred_strength.reshape( (pred_strength.shape[0], pred_strength.shape[2])) assert pred_strength.shape == test_strength.shape print "evaluate performance of the system..." accu, mae, rmse = evaluate(strength_gold=test_strength, strength_pred=pred_strength) print "%s over %s--->accuracy: %s, mae: %s, rmse: %s\n\n" % ( model_name, dataset_name, accu, mae, rmse) pred_strengths_lines = [] for strength in pred_strength: pred_strengths_lines.append('%s\n' % ' '.join([str(ele) for ele in strength]))
def VGGLikeModel(weights='ResNetLike-125k.wts.h5', input_shape=[1024, 2], classes=24, **kwargs): if weights is not None and not (os.path.exists(weights)): raise ValueError('The `weights` argument should be either ' '`None` (random initialization), ' 'or the path to the weights file to be loaded.') dr = 0.5 # dropout rate (%) model = models.Sequential() tap = 8 model.add( Conv1D(input_shape=input_shape, filters=64, kernel_size=tap, padding='same', activation="relu", name="conv1", kernel_initializer='glorot_uniform')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add( Conv1D(filters=64, kernel_size=tap, padding='same', activation="relu", name="conv2", kernel_initializer='glorot_uniform')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add( Conv1D(filters=64, kernel_size=tap, padding='same', activation="relu", name="conv3", kernel_initializer='glorot_uniform')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add( Conv1D(filters=64, kernel_size=tap, padding='same', activation="relu", name="conv4", kernel_initializer='glorot_uniform')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add( Conv1D(filters=64, kernel_size=tap, padding='same', activation="relu", name="conv5", kernel_initializer='glorot_uniform')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add( Conv1D(filters=64, kernel_size=tap, padding='same', activation="relu", name="conv6", kernel_initializer='glorot_uniform')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add( Conv1D(filters=64, kernel_size=tap, padding='same', activation="relu", name="conv7", kernel_initializer='glorot_uniform')) model.add(MaxPooling1D(pool_size=2, strides=2)) model.add(Flatten()) model.add( Dense(128, activation='selu', kernel_initializer='he_normal', name="dense1", kernel_regularizer=regularizers.l2(0.01))) model.add(Dropout(dr)) model.add( Dense(128, activation='selu', kernel_initializer='he_normal', name="dense2")) model.add(Dropout(dr)) model.add( Dense(classes, kernel_initializer='he_normal', name="dense3", kernel_regularizer=regularizers.l2(0.01))) model.add(Activation('softmax')) # model.add(Reshape((classes,))) # Load weights. if weights is not None: model.load_weights(weights) return model
def trainModel(self): ''' Definición y compilación del modelo. ''' input_size = self.max_length #Inputs y embeddings sequence_input = Input(shape=(input_size, ), dtype='float64', name="input_data") sequence_input_lengths = Input(shape=(1, ), dtype='int32', name="input_sequence_lengths") #Definimos función de pérdida -> en función de sequence_input_lengths self.loss_object = Lossfunction(self.batch_size, sequence_input_lengths) embedding_layer = Embedding(self.word_embeddings.shape[0], self.word_embeddings.shape[1], weights=[self.word_embeddings], trainable=False, input_length=input_size, name="embeddings") #Trainable false embedded_sequence = embedding_layer(sequence_input) #Una primera capa con bi-LSTM x = Bidirectional(LSTM(units=self.num_units, return_sequences=True))(embedded_sequence) #Primera convolución x = Conv1D(filters=self.filter_1, kernel_size=self.kernel_1, padding="same", activation="tanh", name="first_convolution")(x) x = MaxPooling1D(pool_size=2, strides=1, padding="same", name="first_max_pooling")(x) x = Dropout(self.dropout_1, name="first_dropout")(x) #Segunda x = Conv1D(filters=self.filter_2, kernel_size=self.kernel_2, padding="same", activation="tanh", name="second_convolution")(x) x = MaxPooling1D(pool_size=2, strides=1, padding="same", name="second_max_pooling")(x) x = Dropout(self.dropout_2, name="second_dropout")(x) #Última capa #preds = TimeDistributed(Dense(4, name = "last_layer"))(x) #preds = Dense(4, activation = "tanh", name = "last_layer")(x) preds = Dense(4, name="last_layer")(x) #Creamos el modelo self.model = Model(input=[sequence_input, sequence_input_lengths], output=[preds]) self.model.summary() self.model.compile(loss=self.loss_object.loss, optimizer='adam', metrics=['accuracy'])
NUM_EPOCHS = 20 EMBEDDING_SIZE = 128 HIDDEN_LAYER_SIZE = 64 """embedding_layer = Embedding(vocab_size + 1 , EMBEDDING_DIM, weights = word_dict, input_length = max_length, trainable=False)""" model = Sequential() model.add( Conv1D(filters=32, kernel_size=3, padding='same', activation='relu', input_shape=input_shape)) model.add(MaxPooling1D(pool_size=2)) model.add(Dropout(0.2)) # 1 layer of 100 units in the hidden layers of the LSTM cells model.add(LSTM(64)) model.add(Dense(5, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # 模型訓練 checkpoint = ModelCheckpoint('sentiment-analysis-model', verbose=1, monitor='loss', save_best_only=True, mode='min') early_stopping = EarlyStopping(monitor='val_loss', min_delta=0,
def build_model(num_classes, input_dim=(500, 64, 1), epochs=100, bs=32, lr=1e-3, init='normal'): #1 CONV => CONV => POOL model = Sequential() model.add( Conv1D(64, 3, padding='same', activation='tanh', input_shape=input_dim)) model.add(BatchNormalization(axis=-1)) model.add(Conv1D(64, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(MaxPooling2D(pool_size=2)) #2 CONV => CONV => POOL model.add(Conv1D(128, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(Conv1D(128, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(MaxPooling1D(pool_size=2)) #3 CONV => CONV => CONV => POOL model.add(Conv1D(256, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(Conv1D(256, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(Conv1D(256, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(MaxPooling1D(pool_size=2)) model.add(Dropout(0.1)) #4 CONV => CONV => CONV => POOL model.add(Conv1D(512, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(Conv1D(512, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(Conv1D(512, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(MaxPooling1D(pool_size=2)) model.add(Dropout(0.1)) #5 CONV => CONV => CONV => POOL model.add(Conv1D(512, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(Conv1D(512, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(Conv1D(512, 3, padding='same', activation='relu')) model.add(BatchNormalization(axis=-1)) model.add(MaxPooling1D(pool_size=2)) model.add(Dropout(0.1)) #6 DENSE-1 model.add(Dense(14 * 4 * 512)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.5)) #7 DENSE-2 model.add(Dense(64 * 64)) model.add(Activation('relu')) model.add(BatchNormalization()) model.add(Dropout(0.5)) # DENSE-3 => OUTPUT model.add(Dense(num_classes)) model.add(Activation('softmax')) # COMPILE MODEL opt = Adam(lr=lr, decay=lr / epochs) model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"]) return model