def predict(self, sentence):
     xs = []
     # tokens = [w.lower() for w in word_tokenize(sentence)]
     tokens = [w for w in word_tokenize(sentence)]
     wid = [
         self.word2idx[token] if token in self.word2idx else 1
         for token in tokens
     ]
     xs.append(wid)
     x = pad_sequences(xs, self.max_len)
     output = self.model.predict(x)
     return output[0]
def fit_text(data_file_path, label_type, max_vocab_size=None):
    if max_vocab_size is None:
        max_vocab_size = 5000

    counter = collections.Counter()
    file = open(data_file_path, mode='rt', encoding='utf8')
    next(file)  # skip header
    max_len = 0
    labels = dict()

    for line in file:
        lst = line.strip().split(',')
        sentence = lst[0]

        if label_type == 'Predicate':
            label = lst[1]
        elif label_type == 'FrameNet':
            label = lst[2]

        # tokens = [x.lower() for x in word_tokenize(sentence)]
        tokens = [x for x in word_tokenize(sentence)]  # Cased word 유지
        for token in tokens:
            counter[token] += 1
        max_len = max(max_len, len(tokens))

        if label not in labels:  # 라벨에 0부터 번호 부여(라벨당 개수 세는 것이 아님)
            labels[label] = len(labels)
    file.close()

    word2idx = collections.defaultdict(int)
    for idx, word in enumerate(counter.most_common(max_vocab_size)):
        word2idx[word[0]] = idx
    idx2word = {v: k for k, v in word2idx.items()}
    vocab_size = len(word2idx) + 1

    model = dict()

    model['word2idx'] = word2idx
    model['idx2word'] = idx2word
    model['vocab_size'] = vocab_size
    model['max_len'] = max_len
    model['labels'] = labels

    return model
    def fit(self,
            text_data_model,
            text_label_pairs,
            model_dir_path,
            batch_size=None,
            epochs=None,
            test_size=None,
            random_state=None):
        if batch_size is None:
            batch_size = 64
        if epochs is None:
            epochs = 20
        if test_size is None:
            test_size = 0.3
        if random_state is None:
            random_state = 42

        self.config = text_data_model
        self.idx2word = self.config['idx2word']
        self.word2idx = self.config['word2idx']
        self.max_len = self.config['max_len']
        self.vocab_size = self.config['vocab_size']
        self.labels = self.config['labels']

        np.save(self.get_config_file_path(model_dir_path), self.config)

        self.create_model()
        json = self.model.to_json()
        open(self.get_architecture_file_path(model_dir_path), 'w').write(json)

        xs = []
        ys = []
        for text, label in text_label_pairs:
            # tokens = [x.lower() for x in word_tokenize(text)]
            tokens = [x for x in word_tokenize(text)]
            wid_list = list()
            for w in tokens:
                wid = 0
                if w in self.word2idx:
                    wid = self.word2idx[w]
                wid_list.append(wid)
            xs.append(wid_list)
            ys.append(self.labels[str(label)])

        X = pad_sequences(xs, maxlen=self.max_len)
        Y = np_utils.to_categorical(ys, len(self.labels))

        x_train, x_test, y_train, y_test = train_test_split(
            X, Y, test_size=test_size, stratify=Y, random_state=random_state)

        print('===========================================')
        print('Below is the shape of train/test dataset.')
        print('===========================================')
        print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)
        print('===========================================')

        weight_file_path = self.get_weight_file_path(model_dir_path)

        checkpoint = ModelCheckpoint(weight_file_path)

        print('===========================================')
        print('======== Now we are on training... ========')
        print('===========================================')

        history = self.model.fit(x=x_train,
                                 y=y_train,
                                 batch_size=batch_size,
                                 epochs=epochs,
                                 validation_data=(x_test, y_test),
                                 callbacks=[checkpoint],
                                 verbose=1)

        self.model.save_weights(weight_file_path)

        np.save(model_dir_path + '/' + WordVecCnn.model_name + '-history.npy',
                history.history)

        # score = self.model.evaluate(x=x_test, y=y_test, batch_size=batch_size, verbose=1)
        # print('score: ', score[0])
        # print('accuracy: ', score[1])
        # print('f1: ', score[2])
        # print('precision: ', score[3])
        # print('recall: ', score[4])

        return history
    def fit(self,
            text_data_model,
            text_label_pairs,
            model_dir_path,
            test_size=None,
            random_state=None,
            epochs=None,
            batch_size=None):
        if epochs is None:
            epochs = 20
        if batch_size is None:
            batch_size = 32
        if test_size is None:
            test_size = 0.3
        if random_state is None:
            random_state = 42

        self.config = text_data_model
        self.idx2word = self.config['idx2word']
        self.word2idx = self.config['word2idx']
        self.max_len = self.config['max_len']
        self.vocab_size = self.config['vocab_size']
        self.labels = self.config['labels']

        verbose = 1

        config_file_path = WordVecMultiChannelCnn.get_config_file_path(
            model_dir_path)
        np.save(config_file_path, text_data_model)

        max_input_tokens = len(self.word2idx)
        self.model = self.define_model(self.max_len, max_input_tokens)
        open(self.get_architecture_file_path(model_dir_path),
             'wt').write(self.model.to_json())

        xs = []
        ys = []
        for text, label in text_label_pairs:
            # tokens = [x.lower() for x in word_tokenize(text)]
            tokens = [x for x in word_tokenize(text)]
            wid_list = list()
            for w in tokens:
                wid = 0
                if w in self.word2idx:
                    wid = self.word2idx[w]
                wid_list.append(wid)
            xs.append(wid_list)
            ys.append(self.labels[str(label)])

        X = pad_sequences(xs, maxlen=self.max_len)
        Y = np_utils.to_categorical(ys, len(self.labels))

        weight_file_path = WordVecMultiChannelCnn.get_weight_file_path(
            model_dir_path)
        checkpoint = ModelCheckpoint(weight_file_path)

        x_train, x_test, y_train, y_test = train_test_split(
            X, Y, test_size=test_size, stratify=Y, random_state=random_state)

        print('===========================================')
        print('Below is the shape of train/test dataset.')
        print('===========================================')
        print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)
        print('===========================================')

        print()

        print('===========================================')
        print('======== Now we are on training... ========')
        print('===========================================')

        history = self.model.fit([x_train, x_train, x_train],
                                 y_train,
                                 epochs=epochs,
                                 batch_size=batch_size,
                                 validation_data=([x_test, x_test,
                                                   x_test], y_test),
                                 verbose=verbose,
                                 callbacks=[checkpoint])
        # save the model
        self.model.save(weight_file_path)

        np.save(
            model_dir_path + '/' + WordVecMultiChannelCnn.model_name +
            '-history.npy', history.history)

        return history