def create_model(train=True): if train: (train_x, chars_x, train_y, word_maxlen, char_maxlen, x_length), ( test_x, test_chars_x, test_y, word_maxlen, char_maxlen, y_length), \ (word_len, char_len, vocab, chars_vocab, chunk_tags, embedding_weights) = process_data.load_cnn_data() else: with open('model/chars-config.pkl', 'rb') as inp: (word_len, char_len, vocab, chars_vocab, chunk_tags, embedding_weights) = pickle.load(inp) # model = Sequential() word_in = Input(shape=(word_len, ), name='word_in') # model.add(Embedding(len(vocab) + 1, EMBED_DIM, weights=[embedding_weights], mask_zero=True)) # Random embedding # words embedding embed_words = Embedding(len(vocab) + 1, EMBED_DIM, mask_zero=True, name='words_embedding')(word_in) # character embedding char_in = Input(shape=( word_len, 1, ), name='char_in') embed_chars = TimeDistributed( Embedding(len(chars_vocab) + 2, 100, mask_zero=False, name='char_embedding'))(char_in) # char_enc = TimeDistributed(LSTM(units=20, return_sequences=False, recurrent_dropout=0.5))(embed_chars) dropout = Dropout(0.3, name='char_dropout')(embed_chars) conv1d_out = TimeDistributed( Conv1D(kernel_size=3, filters=100, padding='same', activation='tanh', strides=1, name='cov1d'))(dropout) maxpool_out = TimeDistributed(MaxPooling1D(1, name='max_pooling'))(conv1d_out) char = TimeDistributed(Flatten(name='flatten'))(maxpool_out) char = Dropout(0.3)(char) x = concatenate([embed_words, char]) x = Bidirectional( LSTM(BiRNN_UNITS // 2, recurrent_dropout=0.1, return_sequences=True, name='LSTM'))(x) x = Dropout(0.7)(x) crf = CRF(len(chunk_tags), sparse_target=True) out = crf(x) model = Model([word_in, char_in], out) model.summary() model.compile('adam', loss=crf.loss_function, metrics=[crf.accuracy]) if train: return model, (train_x, chars_x, train_y, word_len), (test_x, test_chars_x, test_y, y_length), (vocab, chunk_tags) else: return model, (vocab, chunk_tags)
import pickle import keras import numpy as np from sklearn_crfsuite.metrics import flat_classification_report import process_data import cnn_bilsm_crf_model EPOCHS = 10 model, (train_x, chars_x, train_y, word_len), (test_x, test_chars_x, test_y, length), (vocab, chunk_tags) = cnn_bilsm_crf_model.create_model() dev_x, dev_chars_x, dev_y, _, _, dev_length = process_data.load_cnn_data( use_dev=True) # train model # split = 7000 chars_x = np.array([[[ch] for ch in s] for s in chars_x]) test_chars_x = np.array([[[ch] for ch in s] for s in test_chars_x]) dev_chars_x = np.array([[[ch] for ch in s] for s in dev_chars_x]) # # train_x = train_x[:100] # chars_x = chars_x[:100] # train_y = train_y[:100] # test_x = test_x[:100] # test_chars_x = test_chars_x[:100] # test_y = test_y[:100] history = model.fit(
import matplotlib.pyplot as plt import pickle import yaml import process_data as pro import batch import cnn np.random.seed(0) tf.set_random_seed(0) # yaml形式の設定を読み込む f = open("settings.yml", encoding='UTF-8') settings = yaml.load(f) image, ratio = pro.load_cnn_data(0) print (image.shape, ratio.shape) print(len(image)) num_train = settings["num_train"] ##訓練用データ数 num_validate = settings["num_validate"] num_test = settings["num_test"] train_x = image[:num_train] val_x = image[num_train:num_train + num_validate] test_x = image[num_train + num_validate:num_train + num_validate + num_test] train_t = ratio[:num_train] val_t = ratio[num_train:num_train + num_validate] test_t = ratio[num_train + num_validate:num_train + num_validate + num_test] print (train_x.shape, val_x.shape, test_x.shape)