def main(rnn_layer='lstm', word_vecs=None): print "Loading data...", df = sentences_df(SENTENCES_CSV) X, y, word2idx, l_enc = load_dataset(df, pad=True) print X.shape y_binary = to_categorical(y) word_vectors = load_bin_vec(word_vecs, word2idx) add_unknown_words(word_vectors, word2idx) print "Data loaded." labels = np.unique(y) n_labels = labels.shape[0] max_len = X.shape[1] vocab_dim = 300 n_vocab = len(word2idx) + 1 # 0 masking embedding_weights = np.zeros((n_vocab+1, vocab_dim)) for word, index in word2idx.items(): embedding_weights[index,:] = word_vectors[word] skf = StratifiedKFold(y, n_folds=10, shuffle=True, random_state=0) cv_scores = [] for i, (train, test) in enumerate(skf): start_time = time.time() model = create_model(n_vocab, n_labels, vocab_dim, embedding_weights, rnn_layer=rnn_layer) if i == 0: print_summary(model.layers) _, score = train_and_test_model(model, X[train], y_binary[train], X[test], y_binary[test]) cv_scores.append(score) train_time = time.time() - start_time print "fold %i/10 - time: %.2f s - acc: %.4f on %i samples" % \ (i+1, train_time, score, len(test)) print "avg cv acc: %.4f" % np.mean(cv_scores)
def train(label_set='full', drop_unk=False, word_vecs=None, setup_only=False, layer_sizes=[512,256], pool_mode='sum'): print "Loading data..." df = sentences_df(SENTENCES_CSV, labels=label_set, drop_unk=drop_unk) X, y, word2idx, l_enc = load_dataset(df, pad=True) print "X shape:", X.shape y_orig = y y_binary = to_categorical(y) labels = np.unique(y_orig) nb_labels = labels.shape[0] if drop_unk: label_set_str = label_set + ' (-unk)' else: label_set_str = label_set print "Number of labels: %i [%s]" % (nb_labels, label_set_str) if nb_labels > 2: y = y_binary maxlen = X.shape[1] vocab_size = len(word2idx) + 1 # 0 masking if pretrained_embeddings is True: word_vectors = load_bin_vec(word_vecs, word2idx) add_unknown_words(word_vectors, word2idx) embedding_weights = np.zeros((vocab_size+1, emb_dim)) for word, index in word2idx.items(): embedding_weights[index,:] = word_vectors[word] else: embedding_weights = None print "Data loaded." skf = StratifiedKFold(y_orig, n_folds=10, shuffle=True, random_state=0) cv_scores = [] for i, (train, test) in enumerate(skf): start_time = time.time() nn = None nn = EnsembleNN(vocab_size, nb_labels, emb_dim, maxlen, embedding_weights, filter_hs, nb_filters, dropout_p, trainable_embeddings, pretrained_embeddings, layer_sizes, pool_mode) if i == 0: print_summary(nn.model.layers) acc = train_and_test_model(nn, X[train], y[train], X[test], y[test], batch_size, nb_epoch, lr, beta_1, beta_2, epsilon) cv_scores.append(acc) train_time = time.time() - start_time print('\nLabel frequencies in y[test]') print_label_frequencies((y_orig[test], l_enc)) y_pred = nn.model.predict(X[test]) y_pred = probas_to_classes(y_pred) c = Counter(y_pred) total = float(len(y_pred)) print('\nLabel frequencies in predict(y[test])') for label, count in c.most_common(): print l_enc.inverse_transform(label), count, count / total print "fold %i/10 - time: %.2f s - acc: %.4f on %i samples" % \ (i+1, train_time, acc, len(test)) print "Avg cv accuracy: %.4f" % np.mean(cv_scores)
def main(): label_set = 'function' df = sentences_df(labels=label_set) labels = np.unique(df.label.values) out = open(GENERATED_TEXT, 'a') for label in labels: label_df = df[df.label == label] sents = label_df['sentence'].values text = '\n'.join(sent for sent in sents) print 'corpus length:', len(text) chars = sorted(list(set(text))) print 'total chars:', len(chars) char_indices = dict((c,i) for i,c in enumerate(chars)) indices_char = dict((i,c) for i,c in enumerate(chars)) # cut text into sequences of maxlen chars maxlen = 60 step = 3 sentences = [] next_chars = [] for i in range(0, len(text) - maxlen, step): sentences.append(text[i: i + maxlen]) next_chars.append(text[i + maxlen]) print 'nb sequences:', len(sentences) print 'Vectorization...' X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool) y = np.zeros((len(sentences), len(chars)), dtype=np.bool) for i, sentence in enumerate(sentences): for t, char in enumerate(sentence): X[i, t, char_indices[char]] = 1 y[i, char_indices[next_chars[i]]] = 1 print 'Build model...' model = Sequential() model.add(GRU(512, return_sequences=True, input_shape=(maxlen, len(chars)), activation='relu')) model.add(Dropout(0.5)) #model.add(GRU(512, return_sequences=True, activation='relu')) #model.add(Dropout(0.5)) model.add(GRU(512, return_sequences=False, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(len(chars), activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam') print_summary(model.layers) log('label: %s\n' % label, out) train_and_generate(600, X, y, model, text, maxlen, chars, indices_char, char_indices, out) model.save_weights('%s.h5' % label) out.close()
def get_model(self): # Model architecture for A-line classification with CNN model = Sequential() model.add(Conv2D(32, (self.patch_size, self.first_kernel_size), padding='valid', activation = 'relu', kernel_initializer = 'he_normal', input_shape=(self.patch_size, self.aline_depth + self.first_kernel_size - 1, 1))) model.add(Reshape((-1, 32))) model.add(MaxPooling1D(pool_size = 2, strides = 2, padding = 'valid')) model.add(BatchNormalization()) model.add(Reshape((-1,32))) model.add(Conv1D(64, self.first_kernel_size - 2, padding = 'same', activation = 'relu', kernel_initializer = 'he_normal')) model.add(MaxPooling1D(pool_size = 2, strides = 2, padding = 'valid')) model.add(BatchNormalization()) model.add(Flatten()) model.add(Dense(100, kernel_initializer = 'he_normal', activation = 'relu')) model.add(Dense(3, kernel_initializer = 'he_normal', activation = 'softmax')) model.compile(optimizer = Adam(lr = 1e-4), loss = 'categorical_crossentropy', metrics = ['accuracy']) print_summary(model) return model
def main(rnn_layer='lstm', word_vecs=None): print "Loading data...", df = sentences_df(SENTENCES_CSV) X, y, word2idx, l_enc = load_dataset(df, pad=True) print X.shape y_binary = to_categorical(y) word_vectors = load_bin_vec(word_vecs, word2idx) add_unknown_words(word_vectors, word2idx) print "Data loaded." labels = np.unique(y) n_labels = labels.shape[0] max_len = X.shape[1] vocab_dim = 300 n_vocab = len(word2idx) + 1 # 0 masking embedding_weights = np.zeros((n_vocab + 1, vocab_dim)) for word, index in word2idx.items(): embedding_weights[index, :] = word_vectors[word] skf = StratifiedKFold(y, n_folds=10, shuffle=True, random_state=0) cv_scores = [] for i, (train, test) in enumerate(skf): start_time = time.time() model = create_model(n_vocab, n_labels, vocab_dim, embedding_weights, rnn_layer=rnn_layer) if i == 0: print_summary(model.layers) _, score = train_and_test_model(model, X[train], y_binary[train], X[test], y_binary[test]) cv_scores.append(score) train_time = time.time() - start_time print "fold %i/10 - time: %.2f s - acc: %.4f on %i samples" % \ (i+1, train_time, score, len(test)) print "avg cv acc: %.4f" % np.mean(cv_scores)
def test_print_summary_expand_nested(self): shape = (None, None, 3) def make_model(): x = inputs = keras.Input(shape) x = keras.layers.Conv2D(3, 1)(x) x = keras.layers.BatchNormalization()(x) return keras.Model(inputs, x) x = inner_inputs = keras.Input(shape) x = make_model()(x) x = make_model()(x) inner_model = keras.Model(inner_inputs, x) inputs = keras.Input(shape) model = keras.Model(inputs, inner_model(inputs)) file_name = 'model_2.txt' temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True) fpath = os.path.join(temp_dir, file_name) writer = open(fpath, 'w') def print_to_file(text, end='\n'): print(text, end=end, file=writer) try: layer_utils.print_summary( model, print_fn=print_to_file, expand_nested=True) self.assertTrue(tf.io.gfile.exists(fpath)) writer.close() reader = open(fpath, 'r') lines = reader.readlines() reader.close() self.assertEqual(len(lines), 34) except ImportError: pass
def get_network_model(lr=1e-3): input = Input(shape=(nn_np_in_size[0], nn_np_in_size[1], nn_in_chnls)) drop0 = Dropout(0.25)(input) conv1 = Conv2D(16, (5, 5), activation='relu', padding='same', kernel_initializer='he_normal')(drop0) conv1 = Conv2D(16, (5, 5), activation='relu', padding='same', kernel_initializer='he_normal')(conv1) pool1 = MaxPooling2D(pool_size=(2, 2))(conv1) drop1 = Dropout(0.25)(pool1) # norm1 = BatchNormalization()(drop1) conv2 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(drop1) conv2 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(conv2) pool2 = MaxPooling2D(pool_size=(2, 2))(conv2) drop2 = Dropout(0.25)(pool2) conv3 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer='he_normal')(drop2) up7 = concatenate([ Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(conv3), conv2 ], axis=3) conv7 = Conv2D(32, (3, 3), activation='relu', padding='same')(up7) conv7 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv7) drop2 = Dropout(0.25)(conv7) up8 = concatenate([ Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(drop2), conv1 ], axis=3) conv8 = Conv2D(16, (5, 5), activation='relu', padding='same')(up8) conv8 = Conv2D(16, (5, 5), activation='relu', padding='same')(conv8) # norm2 = BatchNormalization()(drop2) # conv3 = Conv2D(64,(3,3),activation='relu',padding='same', kernel_initializer = 'he_normal')(drop2) # conv3 = Conv2D(64,(3,3),activation='relu',padding='same', kernel_initializer = 'he_normal')(conv3) drop3 = Dropout(0.5)(conv8) out = Conv2D(nn_out_chnls, (1, 1), activation='sigmoid', padding='same', kernel_initializer='he_normal')(drop3) model = Model(input, out) # optimizer = SGD(lr=lr, momentum=0.9, decay=1e-4) optimizer = Adam(lr=lr) #, decay=1e-4) model.compile(optimizer=optimizer, loss=dice_coef_loss, metrics=[]) print_summary(model) # plot_model(model, show_shapes=True) return model
def create_model(self, img_shape=None, use_model='', pop_layers=0): # inputs = Input(shape=img_shape) # concat_axis = 1 # load base VGG model base_model = VGG16(weights="imagenet", include_top=False, input_shape=img_shape) print(' VGG Model loaded.') # base_model.summary() # Freeze the first few layers which we don't want to train # for layer in modelVGG.layers[:16]: # layer.trainable = False concat_axis = 3 base_layer_outputs = {} for layer in base_model.layers: layer.trainable = False base_layer_outputs[layer.name] = layer.get_output_at(0) # print(layer.name) # x = base_model.output x = base_layer_outputs['block5_conv3'] print_summary(base_model) # now upsampling up_conv1 = UpSampling2D(size=(2, 2), name='up_conv1')(x) ch, cw = self.get_crop_shape(base_layer_outputs['block5_conv3'], up_conv1) fw_layer = Cropping2D(cropping=(ch, cw))( base_layer_outputs['block5_conv3']) up_conv = concatenate([up_conv1, fw_layer], axis=concat_axis) up_conv = Dropout(0.2)(up_conv) up_conv = Conv2D(512, (3, 3), activation='elu', padding='same')(up_conv) up_conv = BatchNormalization()(up_conv) up_conv = Conv2D(512, (3, 3), activation='elu', padding='same')(up_conv) up_conv = BatchNormalization()(up_conv) up_conv2 = UpSampling2D(size=(2, 2))(up_conv) ch, cw = self.get_crop_shape(base_layer_outputs['block4_conv3'], up_conv2) fw_layer = Cropping2D(cropping=(ch, cw))( base_layer_outputs['block4_conv3']) up_conv = concatenate([up_conv2, fw_layer], axis=concat_axis) up_conv = Dropout(0.2)(up_conv) up_conv = Conv2D(256, (3, 3), activation='elu', padding='same')(up_conv) up_conv = BatchNormalization()(up_conv) up_conv = Conv2D(256, (3, 3), activation='elu', padding='same')(up_conv) up_conv = BatchNormalization()(up_conv) up_conv3 = UpSampling2D(size=(2, 2))(up_conv) ch, cw = self.get_crop_shape(base_layer_outputs['block3_conv3'], up_conv3) fw_layer = Cropping2D(cropping=(ch, cw))( base_layer_outputs['block3_conv3']) up_conv = concatenate([up_conv3, fw_layer], axis=concat_axis) up_conv = Dropout(0.2)(up_conv) up_conv = Conv2D(128, (3, 3), activation='elu', padding='same')(up_conv) up_conv = BatchNormalization()(up_conv) up_conv = Conv2D(128, (3, 3), activation='elu', padding='same')(up_conv) # output X x X # output = Conv2D(5, (1, 1), activation='sigmoid', padding='same')(up_conv) output = Dense(5, activation='softmax')(up_conv) model = Model(input=base_model.input, output=output) model.summary() return model
# # # x = Deconv2D(int(128 * a), kernel_size=2, strides=2)(x) # # x = Conv2D(int(128 * a), 2, activation='relu', padding='same', kernel_initializer='he_normal')(x) # # x = concatenate([conv2, x], axis=3) # # x = Conv2D(int(128 * a), 3, activation='relu', padding='same', kernel_initializer='he_normal')(x) # # x = Conv2D(int(128 * a), 3, activation='relu', padding='same', kernel_initializer='he_normal')(x) # # # x = Deconv2D(int(64 * a), kernel_size=2, strides=2)(x) # # x = Conv2D(int(64 * a), 2, activation='relu', padding='same', kernel_initializer='he_normal')(x) # # x = concatenate([conv1, x], axis=3) # # x = Conv2D(int(64 * a), 3, activation='relu', padding='same', kernel_initializer='he_normal')(x) # # x = Conv2D(int(64 * a), 3, activation='relu', padding='same', kernel_initializer='he_normal')(x) # x = Conv2D(2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(x) x = Conv2D(2, 1, activation='softmax')(x) _output = x model = Model(_input, _output) return model def create(input_shape, base=None): return unet(input_shape) if __name__ == '__main__': from keras.utils.layer_utils import print_summary net = create(input_shape=(240, 320, 3)) print_summary(net)
def train(model_type='parallel', label_set='full', drop_unk=False, word_vecs=None, setup_only=False): print "Loading data..." df = sentences_df(SENTENCES_CSV, labels=label_set, drop_unk=drop_unk) X, y, word2idx, l_enc = load_dataset(df, pad=True) print "X shape:", X.shape y_orig = y y_binary = to_categorical(y) labels = np.unique(y_orig) nb_labels = labels.shape[0] if drop_unk: label_set_str = label_set + ' (-unk)' else: label_set_str = label_set print "Number of labels: %i [%s]" % (nb_labels, label_set_str) if nb_labels > 2: y = y_binary maxlen = X.shape[1] vocab_size = len(word2idx) + 1 # 0 masking if pretrained_embeddings is True: word_vectors = load_bin_vec(word_vecs, word2idx) add_unknown_words(word_vectors, word2idx) embedding_weights = np.zeros((vocab_size + 1, emb_dim)) for word, index in word2idx.items(): embedding_weights[index, :] = word_vectors[word] else: embedding_weights = None print "Data loaded." if setup_only: cnn = create_model(vocab_size, nb_labels, emb_dim, maxlen, embedding_weights, filter_hs, nb_filters, dropout_p, trainable_embeddings, pretrained_embeddings, model_type=model_type) return { 'X': X, 'y': y, 'word2idx': word2idx, 'l_enc': l_enc, 'y_binary': y_binary, 'labels': labels, 'nb_labels': nb_labels, 'maxlen': maxlen, 'emb_dim': emb_dim, 'vocab_size': vocab_size, 'embedding_weights': embedding_weights, 'cnn': cnn } params = [('filter_hs', filter_hs), ('nb_filters', nb_filters), ('dropout_p', dropout_p), ('trainable_embeddings', trainable_embeddings), ('pretrained_embeddings', pretrained_embeddings), ('batch_size', batch_size), ('nb_epoch', nb_epoch), ('lr', lr), ('beta_1', beta_1), ('beta_2', beta_2), ('epsilon', epsilon)] print "\nModel type: %s" % model_type for (name, value) in params: print name + ':', value skf = StratifiedKFold(y_orig, n_folds=10, shuffle=True, random_state=0) cv_scores = [] for i, (train, test) in enumerate(skf): start_time = time.time() cnn = None cnn = create_model(vocab_size, nb_labels, emb_dim, maxlen, embedding_weights, filter_hs, nb_filters, dropout_p, trainable_embeddings, pretrained_embeddings, model_type=model_type) if i == 0: print_summary(cnn.model.layers) acc = train_and_test_model(cnn, X[train], y[train], X[test], y[test], batch_size, nb_epoch, lr, beta_1, beta_2, epsilon) cv_scores.append(acc) train_time = time.time() - start_time print('\nLabel frequencies in y[test]') print_label_frequencies((y_orig[test], l_enc)) y_pred = cnn.model.predict(X[test]) y_pred = probas_to_classes(y_pred) c = Counter(y_pred) total = float(len(y_pred)) print('\nLabel frequencies in predict(y[test])') for label, count in c.most_common(): print l_enc.inverse_transform(label), count, count / total print "fold %i/10 - time: %.2f s - acc: %.4f on %i samples" % \ (i+1, train_time, acc, len(test)) print "Avg cv accuracy: %.4f" % np.mean(cv_scores)
def test_print_summary_expand_nested(self): shape = (None, None, 3) def make_model(): x = inputs = keras.Input(shape) x = keras.layers.Conv2D(3, 1)(x) x = keras.layers.BatchNormalization()(x) return keras.Model(inputs, x) x = inner_inputs = keras.Input(shape) x = make_model()(x) inner_model = keras.Model(inner_inputs, x) inputs = keras.Input(shape) model = keras.Model(inputs, inner_model(inputs)) file_name = 'model_2.txt' temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True) fpath = os.path.join(temp_dir, file_name) writer = open(fpath, 'w') def print_to_file(text): print(text, file=writer) try: layer_utils.print_summary( model, print_fn=print_to_file, expand_nested=True) self.assertTrue(tf.io.gfile.exists(fpath)) writer.close() reader = open(fpath, 'r') lines = reader.readlines() reader.close() check_str = ( 'Model: "model_2"\n' '_________________________________________________________________\n' ' Layer (type) Output Shape Param # \n' '=================================================================\n' ' input_3 (InputLayer) [(None, None, None, 3)] 0 \n' ' \n' ' model_1 (Functional) (None, None, None, 3) 24 \n' '|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n' '| input_1 (InputLayer) [(None, None, None, 3)] 0 |\n' '| |\n' '| model (Functional) (None, None, None, 3) 24 |\n' '||¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯||\n' '|| input_2 (InputLayer) [(None, None, None, 3)] 0 ||\n' '|| ||\n' '|| conv2d (Conv2D) (None, None, None, 3) 12 ||\n' '|| ||\n' '|| batch_normalization (BatchN (None, None, None, 3) 12 ||\n' '|| ormalization) ||\n' '|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n' '¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n' '=================================================================\n' 'Total params: 24\n' 'Trainable params: 18\n' 'Non-trainable params: 6\n' '_________________________________________________________________\n') fin_str = '' for line in lines: fin_str += line self.assertIn(fin_str, check_str) self.assertEqual(len(lines), 25) except ImportError: pass
# model.compile(loss = "categorical_crossentropy", # optimizer = optimizers.SGD(lr=0.00003, momentum=0.9, nesterov=True, decay=1e-6), # metrics=["accuracy"], # decay=0.0005) # model.compile(loss = "mean_absolute_error", # optimizer = optimizers.SGD(lr=0.0001, momentum=0.9, nesterov=True), # metrics=["accuracy"], # decay=0.0005) model.compile(loss='categorical_crossentropy', optimizer=optimizers.Nadam(lr=0.00001), metrics=['accuracy']) print_summary(model) plot_model(model, to_file='model.pdf', show_shapes=True) # ===== first stage history = model.fit_generator(train_generator, steps_per_epoch=1000, epochs=epochs1, validation_data=validation_generator, validation_steps=100, initial_epoch=0, pickle_safe=True, verbose=1, callbacks=[checkpoint, tensorboard]) save_history(history, 'pre')
autoencoder.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(300,))) autoencoder.add(Dropout(0.5)) autoencoder.add(Dense(512)) autoencoder.add(BatchNormalization()) autoencoder.add(PReLU()) autoencoder.add(Dropout(0.5)) autoencoder.add(RepeatVector(maxlen)) autoencoder.add(TimeDistributed(Dense(300))) autoencoder.add(BatchNormalization()) autoencoder.add(PReLU()) autoencoder.add(Dropout(0.5)) autoencoder.add(TimeDistributed(Dense(vocab_size, activation='softmax'))) autoencoder.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print_summary(autoencoder.layers) generator = minibatches(X_train, word2idx) samples_per_epoch = X_train.shape[0] autoencoder.fit_generator(generator, samples_per_epoch=samples_per_epoch, nb_epoch=2) autoencoder.save_weights('autoencoder.h5')
def generate_model(input_shape): input_img = x = Input(shape=input_shape, name='input_img') x = Dropout(0.25)(x) x = Convolution2D(32, 3, 3, border_mode='same')(input_img) x = Convolution2D(32, 3, 3, border_mode='same')(x) x = Maxout2D(16, 2)(x) pool1 = x = MaxPooling2D((2, 2), name='pool1')(x) x = Dropout(0.25)(x) x = Convolution2D(32, 3, 3, border_mode='same')(x) x = Convolution2D(32, 3, 3, border_mode='same')(x) x = Maxout2D(16, 2)(x) pool2 = x = MaxPooling2D((2, 2), name='pool2')(x) x = Dropout(0.25)(x) x = Convolution2D(32, 3, 3, border_mode='same')(x) x = Convolution2D(32, 3, 3, border_mode='same')(x) x = Maxout2D(16, 2)(x) pool3 = x = MaxPooling2D((2, 2), name='pool3')(x) # -- binary presence part x = Dropout(0.25)(x) x = Convolution2D(32, 3, 3, border_mode='same')(x) x = Convolution2D(32, 3, 3, border_mode='same')(x) x = Maxout2D(16, 2)(x) pool4 = x = MaxPooling2D((2, 2), name='pool4')(x) x = Dropout(0.25)(x) x = Convolution2D(32, 3, 3, border_mode='same')(x) x = Convolution2D(32, 3, 3, border_mode='same')(x) x = Maxout2D(16, 2)(x) pool5 = x = MaxPooling2D((2, 2), name='pool5')(x) # Since some images have not mask, the hope is that the innermost units capture this x = Flatten()(pool5) x = Dense(32)(x) x = LeakyReLU()(x) x = Dense(16)(x) x = LeakyReLU()(x) outbin = Dense(1, activation='sigmoid', name='outbin')(x) x = Maxout2D(8, 2)(pool5) outmap5 = Convolution2D(1, 1, 1, border_mode='same', activation='sigmoid', name='outmap5')(x) x = UpSampling2D((2, 2))(x) x = merge([x, pool4], mode='concat', concat_axis=1) x = Convolution2D(16, 3, 3, border_mode='same')(x) x = Maxout2D(8, 2)(x) outmap4 = Convolution2D(1, 1, 1, border_mode='same', activation='sigmoid', name='outmap4')(x) x = UpSampling2D((2, 2))(x) x = merge([x, pool3], mode='concat', concat_axis=1) x = Convolution2D(16, 3, 3, border_mode='same')(x) x = Maxout2D(8, 2)(x) x = UpSampling2D((2, 2))(x) x = merge([x, pool2], mode='concat', concat_axis=1) x = Convolution2D(16, 3, 3, border_mode='same')(x) x = Maxout2D(8, 2)(x) x = UpSampling2D((2, 2))(x) x = merge([x, pool1], mode='concat', concat_axis=1) x = Convolution2D(16, 3, 3, border_mode='same')(x) x = Maxout2D(8, 2)(x) x = UpSampling2D((2, 2))(x) x = Dropout(0.25)(x) x = Convolution2D(8, 3, 3, border_mode='same')(x) x = Convolution2D(8, 3, 3, border_mode='same')(x) outmap = Convolution2D(1, 3, 3, activation='sigmoid', border_mode='same', name='outmap')(x) model = Model(input=input_img, output=[outmap, outmap4, outmap5, outbin]) #sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) #model.compile(optimizer=sgd, loss='binary_crossentropy') #rmsprop = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08) #model.compile(optimizer=rmsprop, loss='binary_crossentropy') metrics = {'outbin': 'accuracy'} model.compile(optimizer='adam', loss='binary_crossentropy', loss_weights=[1., 0.2, 0.05, 0.01], metrics=metrics) print_summary(model.layers) return model
def get_model(self, numpy_matrix_embeddings: numpy.ndarray, **kwargs) -> Model: # get default parameters dropout = kwargs.get('dropout', 0.9) assert isinstance(dropout, float) lstm_layer_size = kwargs.get('lstm_layer_size', 64) assert isinstance(lstm_layer_size, int) # we also need the full input length # nope, we don't need it :) # max_input_length = kwargs.get('max_input_length') # assert max_input_length # this is the standard placeholder tensor for the input sequences; a numpy array of word indices # input_sequence = Input(shape=(max_input_length,), dtype='int32', name='input_sequence') input_sequence = Input(shape=(None, ), dtype='int32', name='input_sequence') print("Embeddings shape:", numpy_matrix_embeddings.shape) # The embedding layer will transform the sequences of integers into vectors of size dimension of embeddings # We also fix the embeddings (not trainable) # # For this model, we have to disable masking as the Reshape layers do not support it embedded = Embedding(numpy_matrix_embeddings.shape[0], numpy_matrix_embeddings.shape[1], weights=[numpy_matrix_embeddings], mask_zero=True, name='embeddings', trainable=False)(input_sequence) # bidirectional LSTM using the neat Keras wrapper lstm_output = Bidirectional(LSTM(lstm_layer_size, return_sequences=True), name="BiLSTM")(embedded) # matrix H # maybe we don't need to reshape? -- nope, we don't need to reshape :) # matrix_h = Reshape((max_input_length, lstm_layer_size * 2), name='H_matrix')(lstm_output) matrix_h = lstm_output # matrix H transposed; we don't need it, it can be transposed in matmul later on # matrix_h_t = Lambda(lambda x: tensorflow.transpose(x, perm=[0, 2, 1]), name='HT_matrix')(matrix_h) # 150 was default param_da = 300 output_tanh_ws1_ht = Dense(units=param_da, activation='tanh', use_bias=False, name='tanh_Ws1_HT')(matrix_h) # 30 was ok param_r = 50 annotation_matrix_a_linear = Dense(units=param_r, activation='linear', use_bias=False, name='A_matrix')(output_tanh_ws1_ht) # 0.1 was ok too # the longer the coefficient, the more only blocks at the beginning are shown # penalization_coefficient = 0.5 # penalization_coefficient = 1 # this kills the performance :( # so without penalization we get to almost 80 percent!! :) penalization_coefficient = 0 # now do the softmax over rows, not the columns annotation_matrix_a = Lambda( lambda x: tensorflow.nn.softmax(x), name='A_softmax', activity_regularizer=PRegularizer(penalization_coefficient))( annotation_matrix_a_linear) # multiplication: easier to use Lambda than Multiply matrix_m = Lambda( lambda x: tensorflow.matmul(x[0], x[1], transpose_a=True), name='M_matrix')([annotation_matrix_a, matrix_h]) # and flatten dense_representation = Flatten()(matrix_m) # classification output = Dense(2, activation='softmax', name="Output_dense")(dense_representation) model = Model(inputs=[input_sequence], outputs=output) print_summary(model) model.compile('adam', loss=categorical_crossentropy) print("Model compiled") debug_graph = False if debug_graph: from tensorflow.python.keras.utils import plot_model plot_model(model, to_file='/tmp/model.png', show_shapes=True) return model
def test_print_summary_layer_range_with_expand_nested(self): shape = (None, None, 3) def make_model(): x = inputs = keras.Input(shape, name="input_2") x = keras.layers.Conv2D(3, 1)(x) x = keras.layers.BatchNormalization()(x) return keras.Model(inputs, x, name="2nd_inner") x = inner_inputs = keras.Input(shape, name="input_1") x = make_model()(x) inner_model = keras.Model(inner_inputs, x, name="1st_inner") inputs = keras.Input(shape, name="input_3") model = keras.Model(inputs, inner_model(inputs)) file_name = "model_8.txt" temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True) fpath = os.path.join(temp_dir, file_name) writer = open(fpath, "w") def print_to_file(text): print(text, file=writer) try: layer_utils.print_summary( model, print_fn=print_to_file, expand_nested=True, layer_range=["1st_inner", "1st_inner"], ) layer_utils.print_summary( model, expand_nested=True, layer_range=["1st_inner", "1st_inner"], ) self.assertTrue(tf.io.gfile.exists(fpath)) writer.close() reader = open(fpath, "r") lines = reader.readlines() reader.close() check_str = ( 'Model: "model"\n' "_________________________________________________________________\n" # noqa: E501 " Layer (type) Output Shape Param # \n" # noqa: E501 "=================================================================\n" # noqa: E501 " 1st_inner (Functional) (None, None, None, 3) 24 \n" # noqa: E501 "|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n" # noqa: E501 "| input_1 (InputLayer) [(None, None, None, 3)] 0 |\n" # noqa: E501 "| |\n" # noqa: E501 "| 2nd_inner (Functional) (None, None, None, 3) 24 |\n" # noqa: E501 "||¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯||\n" # noqa: E501 "|| input_2 (InputLayer) [(None, None, None, 3)] 0 ||\n" # noqa: E501 "|| ||\n" # noqa: E501 "|| conv2d (Conv2D) (None, None, None, 3) 12 ||\n" # noqa: E501 "|| ||\n" # noqa: E501 "|| batch_normalization (BatchN (None, None, None, 3) 12 ||\n" # noqa: E501 "|| ormalization) ||\n" # noqa: E501 "|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n" # noqa: E501 "¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n" # noqa: E501 "=================================================================\n" # noqa: E501 "Total params: 24\n" "Trainable params: 18\n" "Non-trainable params: 6\n" "_________________________________________________________________\n" # noqa: E501 ) check_lines = check_str.split( "\n")[:-1] # Removing final empty string which is not a line fin_str = "".join(lines) self.assertIn(fin_str, check_str) self.assertEqual(len(lines), len(check_lines)) except ImportError: pass
x = Conv2D(16, kernel_size=(3, 3), activation='relu', padding='same')(input_img) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(8, kernel_size=(3, 3), activation='relu', padding='same')(x) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(8, kernel_size=(3, 3), activation='relu', padding='same')(x) encoded = MaxPooling2D((2, 2), padding='same')(x) # at this point the representation is (4, 4, 8) i.e. 128-dimensional x = Conv2D(8, kernel_size=(3, 3), activation='relu', padding='same')(encoded) x = UpSampling2D((2, 2))(x) x = Conv2D(8, kernel_size=(3, 3), activation='relu', padding='same')(x) x = UpSampling2D((2, 2))(x) x = Conv2D(16, kernel_size=(3, 3), activation='relu')(x) x = UpSampling2D((2, 2))(x) decoded = Conv2D(1, kernel_size=(3, 3), activation='sigmoid', padding='same')(x) autoencoder = Model(input_img, decoded) autoencoder.compile(optimizer='adadelta', loss='binary_crossentropy') print_summary(autoencoder) autoencoder.fit(X_train, X_train, epochs=50, batch_size=128, shuffle=True, validation_data=(X_test, X_test))
def SCVAE(expr, patience=30, metric='Silhouette', outliers=False, prefix=None, k=None, label=None, id_map=None, log=True, scale=True, rep=0): expr[expr < 0] = 0.0 if log: expr = np.log2(expr + 1) if scale: for i in range(expr.shape[0]): expr[i, :] = expr[i, :] / np.max(expr[i, :]) if outliers: o = outliers_detection(expr) expr = expr[o == 1, :] if label is not None: label = label[o == 1] if rep > 0: expr_train = np.matlib.repmat(expr, rep, 1) else: expr_train = np.copy(expr) vae_ = VAE(in_dim=expr.shape[1], loss=config['loss']) vae_.vaeBuild() print_summary(vae_.vae) if prefix is not None: plot_model(vae_.vae, to_file=prefix + '_model.eps', show_shapes=True) wait = 0 best_metric = -np.inf epoch = config['epoch'] batch_size = config['batch_size'] res_cur = [] aux_cur = [] for e in range(epoch): print("Epoch %d/%d" % (e + 1, epoch)) loss = vae_.vae.fit(expr_train, expr_train, epochs=1, batch_size=batch_size, shuffle=True) train_loss = -loss.history['loss'][0] #val_loss = -loss.history['val_loss'][0] print("Loss:" + str(train_loss)) #print(h1) if k is None and label is not None: k = len(np.unique(label)) # for r in res: # print("======"+str(r.shape[1])+"========") # #if r.shape[1] == 2: # # r = cart2polar(r) # pred,si = clustering( r,k=k ) # if label is not None: # metrics_ = measure( pred,label ) # metrics_['Silhouette'] = si # cur_metric = train_loss #metrics[metric] if best_metric < cur_metric: best_metric = cur_metric wait = 0 res = vae_.ae.predict(expr) res_cur = res aux_cur = vae_.aux.predict(expr) #if prefix is not None: # model_file = prefix+'_model_weights_'+str(e)+'.h5' # vae_.vae.save_weights(model_file) else: wait += 1 if e > 100 and wait > patience: break ## use best_e for subsequent analysis ## visualization if prefix is not None: for r in res_cur: _, dim = r.shape pic_name = prefix + '_dim' + str(dim) + '.eps' if dim == 2: #r = cart2polar(r) if outliers: o = outliers_detection(r) r_p = r[o == 1] label_p = label[o == 1] else: r_p = np.copy(r) label_p = np.copy(label) fig = print_2D(r_p, label_p, id_map=id_map) fig.savefig(pic_name) else: fig32 = print_heatmap(r, label=label, id_map=id_map) fig32.savefig(pic_name) ## analysis results aux_res = h5py.File(prefix + '_res.h5', mode='w') aux_res.create_dataset(name='AUX', data=aux_cur) aux_res.create_dataset(name='EXPR', data=expr) count = 1 for r in res_cur: aux_res.create_dataset(name='RES' + str(count), data=r) count += 1 aux_res.close() return res_cur
autoencoder.add(Lambda(lambda x: K.sum(x, axis=1), output_shape=(300, ))) autoencoder.add(Dropout(0.5)) autoencoder.add(Dense(512)) autoencoder.add(BatchNormalization()) autoencoder.add(PReLU()) autoencoder.add(Dropout(0.5)) autoencoder.add(RepeatVector(maxlen)) autoencoder.add(TimeDistributed(Dense(300))) autoencoder.add(BatchNormalization()) autoencoder.add(PReLU()) autoencoder.add(Dropout(0.5)) autoencoder.add(TimeDistributed(Dense(vocab_size, activation='softmax'))) autoencoder.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print_summary(autoencoder.layers) generator = minibatches(X_train, word2idx) samples_per_epoch = X_train.shape[0] autoencoder.fit_generator(generator, samples_per_epoch=samples_per_epoch, nb_epoch=2) autoencoder.save_weights('autoencoder.h5')
def train_and_predict(): pretrained_weights_path = args.weights structure_mode = args.structure print_pretty('Creating and compiling model...') network_input_shp = (config.NETWORK_INPUT_H, config.NETWORK_INPUT_W, config.NETWORK_INPUT_C) output_shp = (config.NETWORK_INPUT_H, config.NETWORK_INPUT_W, 1) train_model, infer_model = create_model(input_shape=network_input_shp, lr=1e-4) if structure_mode: print_summary(train_model) plot_model(infer_model, rankdir='LB', show_shapes=True, show_layer_names=True, to_file='train_model.png') return if pretrained_weights_path: train_model.load_weights(args.weights) print_pretty('Loading and preprocessing train data...') train_imgs, train_masks, valid_imgs, valid_masks = robofest_data_get_samples_preprocessed( network_input_shp, output_shp) if len(np.unique(valid_masks)) > 2: print( 'Valid: Preprocessing created mask with more than two binary values' ) exit(1) if len(np.unique(train_masks)) > 2: print( 'Train: Preprocessing created mask with more than two binary values' ) exit(1) print_pretty('Setup data generator...') imgs_valid = valid_imgs imgs_mask_valid = valid_masks imgs_train = train_imgs imgs_mask_train = train_masks print('Train:', imgs_train.shape) print('Valid:', imgs_valid.shape) data_gen_args = dict(rotation_range=5, width_shift_range=0.1, height_shift_range=0.1, zoom_range=0.1, horizontal_flip=True, fill_mode='constant', cval=0) image_datagen = ImageDataGenerator(**data_gen_args) mask_datagen = ImageDataGenerator(**data_gen_args) seed = 1 batch_size = 8 image_datagen.fit(imgs_train, augment=True, seed=seed) mask_datagen.fit(imgs_mask_train, augment=True, seed=seed) print_pretty('Flowing data...') image_generator = image_datagen.flow(imgs_train, batch_size=batch_size, seed=seed) mask_generator = mask_datagen.flow(imgs_mask_train, batch_size=batch_size, seed=seed) print_pretty('Zipping generators...') train_generator = zip(image_generator, mask_generator) print_pretty('Fitting model...') checkpoint_vloss = CustomModelCheckpoint( model_to_save=infer_model, filepath=config.NET_BASENAME + '_ep{epoch:03d}-iou{iou_metrics:.3f}-val_iou{val_iou_metrics:.3f}' + '.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1) train_model.fit_generator(train_generator, steps_per_epoch=20, epochs=10000, verbose=1, validation_data=(imgs_valid, imgs_mask_valid), callbacks=[ ModelCheckpoint('chk/weights_best.h5', monitor='val_iou_metrics', mode='max', save_best_only=True, save_weights_only=False, verbose=1) ])
def create_model(self, img_shape=None, use_model='', pop_layers=0): # use_model could be either 'VGG16' or 'RESNET' or path to a checkpoint. Set pop_layers to the number of layers to strip # inputs = Input(shape=img_shape) # concat_axis = 1 # load base VGG model base_model = VGG16(weights="imagenet", include_top=False, input_shape=img_shape) print(' VGG Model loaded.') # make the structure fully match to the crop-trained net x = base_model.output x = BatchNormalization()(x) x = Dropout(0.25)(x) x = Conv2D(512, (1, 1), activation='elu', padding='valid', name='Kir_1')(x) x = BatchNormalization()(x) x = Conv2D(512, (1, 1), activation='elu', padding='valid', name='Kir_2')(x) x = BatchNormalization()(x) x = Conv2D(6, (3, 3), activation='sigmoid', padding='valid', name='Kir_3')(x) # predictions = Reshape(target_shape=(6,))(x) base_model = Model(input=base_model.input, output=x) # это уже совсем другая base_model # now load weights base_model.load_weights(use_model) # remove un-neded layers for i in range(pop_layers): base_model.layers.pop() print(' Custom Model loaded and trimmed.') # base_model.summary() # Freeze the first few layers which we don't want to train # for layer in modelVGG.layers[:16]: # layer.trainable = False concat_axis = 3 base_layer_outputs = {} for layer in base_model.layers: layer.trainable = False base_layer_outputs[layer.name] = layer.get_output_at(0) # print(layer.name) x = base_model.output print_summary(base_model) # now upsampling up_conv1 = UpSampling2D(size=(2, 2), name='up_conv1')(x) ch, cw = self.get_crop_shape(base_layer_outputs['block5_conv3'], up_conv1) fw_layer = Cropping2D(cropping=(ch, cw))( base_layer_outputs['block5_conv3']) up_conv = concatenate([up_conv1, fw_layer], axis=concat_axis) # up_conv = Dropout(0.2)(up_conv) up_conv = Conv2D(512, (3, 3), activation='elu', padding='same')(up_conv) # up_conv = BatchNormalization()(up_conv) up_conv = Conv2D(512, (3, 3), activation='elu', padding='same')(up_conv) up_conv = BatchNormalization()(up_conv) up_conv2 = UpSampling2D(size=(2, 2))(up_conv) ch, cw = self.get_crop_shape(base_layer_outputs['block4_conv3'], up_conv2) fw_layer = Cropping2D(cropping=(ch, cw))( base_layer_outputs['block4_conv3']) up_conv = concatenate([up_conv2, fw_layer], axis=concat_axis) # up_conv = Dropout(0.2)(up_conv) up_conv = Conv2D(256, (3, 3), activation='elu', padding='same')(up_conv) # up_conv = BatchNormalization()(up_conv) up_conv = Conv2D(256, (3, 3), activation='elu', padding='same')(up_conv) up_conv = BatchNormalization()(up_conv) up_conv3 = UpSampling2D(size=(2, 2))(up_conv) ch, cw = self.get_crop_shape(base_layer_outputs['block3_conv3'], up_conv3) fw_layer = Cropping2D(cropping=(ch, cw))( base_layer_outputs['block3_conv3']) up_conv = concatenate([up_conv3, fw_layer], axis=concat_axis) # up_conv = Dropout(0.2)(up_conv) up_conv = Conv2D(128, (3, 3), activation='elu', padding='same')(up_conv) up_conv = BatchNormalization()(up_conv) up_conv = Conv2D(128, (3, 3), activation='elu', padding='same')(up_conv) # output X x X # output = Conv2D(5, (1, 1), activation='sigmoid', padding='same')(up_conv) output = Dense(5, activation='softmax')(up_conv) model = Model(input=base_model.input, output=output) model.summary() return model
word_vectors = load_bin_vec(word_vecs, word2idx) add_unknown_words(word_vectors, word2idx) embedding_weights = np.zeros((vocab_size, emb_dim)) for word, index in word2idx.items(): embedding_weights[index, :] = word_vectors[word] model = build_model(vocab_size, max_caption_len, vggweights_path, embedding_weights, scene_model=scene_model) if modelweights_path: model.load_weights(modelweights_path) for m in model.layers[0].layers: print_summary(m.layers) print_summary(model.layers) partial_captions_train, partial_captions_test = (partial_captions[:280000], partial_captions[280000:]) imfiles_train, imfiles_test = (imfiles[:280000], imfiles[280000]) next_words_train, next_words_test = (next_words[:280000], next_words[280000:]) scenes_train, scenes_test = (scenes[:280000], scenes[280000:]) samples_per_epoch = partial_captions_train.shape[0] imstream = imstream(imlookup, imfiles_train) capstream = stream(partial_captions_train) nextstream = nextwordstream(next_words_train) scenestream = stream(scenes_train)
model.add(Dropout(0.5)) model.add(TimeDistributed(Dense(vocab_size, activation='softmax'))) model.load_weights(weights_path) for _ in range(6): model.pop() model.add(Dense(nb_labels, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model skf = StratifiedKFold(y_orig, n_folds=10, shuffle=True, random_state=0) cv_scores = [] for i, (train, test) in enumerate(skf): start_time = time.time() nn = None nn = build_model() if i == 0: print_summary(nn.layers) nn.fit(X[train], y[train], nb_epoch=nb_epoch, batch_size=batch_size, validation_split=0.00) score, acc = nn.evaluate(X[test], y[test], batch_size=64) cv_scores.append(acc) train_time = time.time() - start_time print "\nfold %i/10 - time: %.2f s - acc: %.4f on %i samples" % \ (i+1, train_time, acc, len(test)) print "Avg cv accuracy: %.4f" % np.mean(cv_scores)
def test_summary_subclass_model_expand_nested(self): class Sequential(keras.Model): def __init__(self, *args): super().__init__() self.module_list = list(args) if args else [] def call(self, x): for module in self.module_list: x = module(x) return x class Block(keras.Model): def __init__(self): super().__init__() self.module = Sequential( keras.layers.Dense(10), keras.layers.Dense(10), ) def call(self, input_tensor): x = self.module(input_tensor) return x class Base(keras.Model): def __init__(self): super().__init__() self.module = Sequential(Block(), Block()) def call(self, input_tensor): x = self.module(input_tensor) y = self.module(x) return x, y class Network(keras.Model): def __init__(self): super().__init__() self.child = Base() def call(self, inputs): return self.child(inputs) net = Network() inputs = keras.Input(shape=(10,)) outputs = net(inputs) model = keras.models.Model(inputs=inputs, outputs=outputs) file_name = 'model_3.txt' temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True) fpath = os.path.join(temp_dir, file_name) writer = open(fpath, 'w') def print_to_file(text): print(text, file=writer) try: layer_utils.print_summary( model, line_length=120, print_fn=print_to_file, expand_nested=True) self.assertTrue(tf.io.gfile.exists(fpath)) writer.close() reader = open(fpath, 'r') lines = reader.readlines() reader.close() # The output content are slightly different for the input shapes between # v1 and v2. if tf.__internal__.tf2.enabled(): self.assertEqual(len(lines), 39) else: self.assertEqual(len(lines), 40) except ImportError: pass
def test_print_summary_without_print_fn(self): model = keras.Sequential( [keras.layers.Dense(5, input_shape=(10, ), name='dense')]) with self.captureWritesToStream(sys.stdout) as printed: layer_utils.print_summary(model) self.assertIn('dense (Dense)', printed.contents())
#Dropout layer prevents overfitting after_dp = Dropout(0.6)(bi_lstm_unit2) #sigmoid activation layer output = Dense(output_dimensions[1], activation='sigmoid', name='activation')(after_dp) #Create mem_model from above defined layers mem_model = Model(inputs=sequence, outputs=output) #Your model is now two stacked bidirectional LSTM cell + a dropout layer + dense layer+ an activation layer. optimizers.Adadelta(lr=0.5, rho=0.95, epsilon=None, decay=0.005) optimizers.Adam(lr=0.1, beta_1=0.6, beta_2=0.099, epsilon=1e-08, decay=0.005, clipnorm = 1., clipvalue = 0.5) mem_model.compile(optimizer = 'Adadelta', loss = 'mean_squared_error', metrics = ['categorical_accuracy']) print('\n\nMemory module loaded. - Phase : ', phase, '\n\n') print_summary(mem_model) print('Beginning Training...') #specify batch_size batch_size = 2 #specify number of training epochs num_epoch = 10 #print("\n\nLoad pretrained weights.") #mem_model = load_model(os.path.join(model_backup_path,"model1.h5")) print("\n\nFitting to model.") my_model = mem_model.fit(np.array(x_train), np.array(y_train), batch_size =batch_size, epochs=num_epoch, verbose =1, validation_data=[np.array(x_test), np.array(y_test)]) print("Model Training complete.")
elif cv: skf = StratifiedKFold(y_orig, n_folds=cv, shuffle=True, random_state=0) cv_scores = [] for i, (train, test) in enumerate(skf): start_time = time.time() nn = None nn = FeedforwardNN(vocab_size, nb_labels, emb_dim, maxlen, layer_sizes, embedding_weights, pool_mode=pool_mode) if i == 0: print_summary(nn.model.layers) nn, acc = train_and_test_model(nn, X[train], y[train], X[test], y[test], batch_size, nb_epoch, lr, beta_1, beta_2, epsilon, val_split=val_split) <<<<<<< HEAD if return_net: d = {'X': X, 'y': y, 'word2idx': word2idx, 'l_enc': l_enc, 'y_binary': y_binary, 'y_orig': y_orig, 'labels': labels, 'nb_labels': nb_labels,
def test_print_summary_expand_nested_show_trainable(self): shape = (None, None, 3) def make_model(): x = inputs = keras.Input(shape, name="input2") untrainable = keras.layers.Conv2D(3, 1) untrainable.trainable = False x = untrainable(x) x = keras.layers.BatchNormalization()(x) return keras.Model(inputs, x) x = inner_inputs = keras.Input(shape, name="input1") x = make_model()(x) inner_model = keras.Model(inner_inputs, x) inputs = keras.Input(shape, name="input3") model = keras.Model(inputs, inner_model(inputs)) file_name = "model_6.txt" temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True) fpath = os.path.join(temp_dir, file_name) writer = open(fpath, "w") def print_to_file(text): print(text, file=writer) try: layer_utils.print_summary( model, print_fn=print_to_file, expand_nested=True, show_trainable=True, ) self.assertTrue(tf.io.gfile.exists(fpath)) writer.close() reader = open(fpath, "r") lines = reader.readlines() reader.close() check_str = ( "Model: " '"model_2"\n____________________________________________________________________________\n' " Layer (type) Output Shape Param # " "Trainable " "\n============================================================================\n" " input3 (InputLayer) [(None, None, None, 3)] 0 Y" " \n" " " "\n model_1 (Functional) (None, None, None, 3) 24 " "Y " "\n|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n|" " input1 (InputLayer) [(None, None, None, 3)] 0 Y" " |\n|" " " "|\n| model (Functional) (None, None, None, 3) 24 " "Y " "|\n||¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯||\n||" " input2 (InputLayer) [(None, None, None, 3)] 0 Y" " ||\n||" " " "||\n|| conv2d (Conv2D) (None, None, None, 3) 12 " "N ||\n||" " " "||\n|| batch_normalization (BatchN (None, None, None, 3) 12 " "Y ||\n|| ormalization)" " " "||\n|¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯|\n¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯\n============================================================================\nTotal" " params: 24\nTrainable params: 6\nNon-trainable params: " "18\n____________________________________________________________________________\n" "____________________________________________________________________________\n" ) fin_str = "" for line in lines: fin_str += line self.assertIn(fin_str, check_str) self.assertEqual(len(lines), 25) except ImportError: pass
def train(model_type='parallel', label_set='full', drop_unk=False, word_vecs=None, setup_only=False): print "Loading data..." df = sentences_df(SENTENCES_CSV, labels=label_set, drop_unk=drop_unk) X, y, word2idx, l_enc = load_dataset(df, pad=True) print "X shape:", X.shape y_orig = y y_binary = to_categorical(y) labels = np.unique(y_orig) nb_labels = labels.shape[0] if drop_unk: label_set_str = label_set + ' (-unk)' else: label_set_str = label_set print "Number of labels: %i [%s]" % (nb_labels, label_set_str) if nb_labels > 2: y = y_binary maxlen = X.shape[1] vocab_size = len(word2idx) + 1 # 0 masking if pretrained_embeddings is True: word_vectors = load_bin_vec(word_vecs, word2idx) add_unknown_words(word_vectors, word2idx) embedding_weights = np.zeros((vocab_size+1, emb_dim)) for word, index in word2idx.items(): embedding_weights[index,:] = word_vectors[word] else: embedding_weights = None print "Data loaded." if setup_only: cnn = create_model(vocab_size, nb_labels, emb_dim, maxlen, embedding_weights, filter_hs, nb_filters, dropout_p, trainable_embeddings, pretrained_embeddings, model_type=model_type) return {'X': X, 'y': y, 'word2idx': word2idx, 'l_enc': l_enc, 'y_binary': y_binary, 'labels': labels, 'nb_labels': nb_labels, 'maxlen': maxlen, 'emb_dim': emb_dim, 'vocab_size': vocab_size, 'embedding_weights': embedding_weights, 'cnn': cnn} params = [('filter_hs',filter_hs), ('nb_filters',nb_filters), ('dropout_p',dropout_p), ('trainable_embeddings',trainable_embeddings), ('pretrained_embeddings',pretrained_embeddings), ('batch_size',batch_size), ('nb_epoch',nb_epoch), ('lr',lr), ('beta_1',beta_1), ('beta_2',beta_2), ('epsilon',epsilon)] print "\nModel type: %s" % model_type for (name, value) in params: print name + ':', value skf = StratifiedKFold(y_orig, n_folds=10, shuffle=True, random_state=0) cv_scores = [] for i, (train, test) in enumerate(skf): start_time = time.time() cnn = None cnn = create_model(vocab_size, nb_labels, emb_dim, maxlen, embedding_weights, filter_hs, nb_filters, dropout_p, trainable_embeddings, pretrained_embeddings, model_type=model_type) if i == 0: print_summary(cnn.model.layers) acc = train_and_test_model(cnn, X[train], y[train], X[test], y[test], batch_size, nb_epoch, lr, beta_1, beta_2, epsilon) cv_scores.append(acc) train_time = time.time() - start_time print('\nLabel frequencies in y[test]') print_label_frequencies((y_orig[test], l_enc)) y_pred = cnn.model.predict(X[test]) y_pred = probas_to_classes(y_pred) c = Counter(y_pred) total = float(len(y_pred)) print('\nLabel frequencies in predict(y[test])') for label, count in c.most_common(): print l_enc.inverse_transform(label), count, count / total print "fold %i/10 - time: %.2f s - acc: %.4f on %i samples" % \ (i+1, train_time, acc, len(test)) print "Avg cv accuracy: %.4f" % np.mean(cv_scores)
def _main_(): config_path = args.conf weights_path = args.weights output_pb_fpath = args.output config_path = args.conf output_dpath = os.path.dirname(output_pb_fpath) if not os.path.isdir(output_dpath): os.makedirs(output_dpath) with open(config_path) as config_buffer: config = json.loads(config_buffer.read()) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.2, allow_growth=True) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) K.set_session(sess) K.set_learning_phase(0) # Swapped as net input -> [H x W x C] net_input_shape = (config['model']['input_shape'][1], config['model']['input_shape'][0], 3) mvnc_model = models.create(base=config['model']['base'], input_shape=net_input_shape) if weights_path: mvnc_model.load_weights(weights_path) print_summary(mvnc_model) model_input_names = [mvnc_model.input.name.split(':')[0]] model_output_names = [mvnc_model.output.name.split(':')[0]] model_outputs = [mvnc_model.output] print('Outputs: {}'.format(model_outputs)) # print('Output shapes: {}'.format(model_output_shapes)) with K.get_session() as sess: graphdef = sess.graph.as_graph_def() # for op in graphdef.node: # print(op.name) dirpath = os.path.join('logs', config['model']['base']) shutil.rmtree(dirpath, ignore_errors=True) if not os.path.isdir(dirpath): os.makedirs(dirpath) writer = tf.summary.FileWriter(dirpath, sess.graph) writer.close() frozen_graph = tf.graph_util.convert_variables_to_constants( sess, graphdef, model_output_names) frozen_graph = tf.graph_util.remove_training_nodes(frozen_graph) frozen_graph_filename = output_pb_fpath with open(frozen_graph_filename, 'wb') as f: f.write(frozen_graph.SerializeToString()) f.close() K.clear_session() ##################################### from subprocess import call graph_fpath = output_pb_fpath + '.graph' print(' Writing to {}'.format(graph_fpath)) process_args = [ "mvNCCompile", output_pb_fpath, "-in", model_input_names[0], "-on", model_output_names[0], "-s", "12", "-o", graph_fpath ] call(process_args) print(' Compiled, check performance') process_args = [ "mvNCCheck", output_pb_fpath, "-in", model_input_names[0], "-on", model_output_names[0], "-s", "12" ] call(process_args) process_args = [ "mvNCProfile", output_pb_fpath, "-in", model_input_names[0], "-on", model_output_names[0], "-s", "12" ] call(process_args)
word_vectors = load_bin_vec(word_vecs, word2idx) add_unknown_words(word_vectors, word2idx) embedding_weights = np.zeros((vocab_size, emb_dim)) for word, index in word2idx.items(): embedding_weights[index,:] = word_vectors[word] model = load_vgg_weights(build_model(vocab_size, max_caption_len, emb_dim, embedding_weights), weights_path) print "VGG 16 weights loaded." print_summary(model.layers) samples_per_epoch = X_sents_train.shape[0] checkpoint = ModelCheckpoint('../../weights.{epoch:02d}.h5') for epoch in range(50): ims = imstream(ims_train) caps = captionstream(X_sents_train) oh_caps = one_hot_captionstream(y_sents_train, word2idx) batches = minibatches(ims, caps, oh_caps) model.fit_generator(minibatches(ims, caps, oh_caps), samples_per_epoch=samples_per_epoch, nb_epoch=1, callbacks=[checkpoint]) """j for X, y in batches: loss, acc = model.train_on_batch(X, y)