Esempio n. 1
0
def create_sentence_files():
    stop_words = set(
        pd.read_csv('./Primary_data/PersianStopWordList.txt', header=None)[0])
    questions = QuickDataFrame.read_csv('./Primary_data/result_filtered.csv',
                                        sep=';')
    topics = QuickDataFrame.read_csv('./Primary_data/topic_vector_Q.csv')

    files = dict()
    for tpc in topics.cols:
        files[tpc + '-p'] = open('./Primary_data/sent_topic/' + tpc + '.p',
                                 'w',
                                 encoding='utf-8')
        files[tpc + '-n'] = open('./Primary_data/sent_topic/' + tpc + '.n',
                                 'w',
                                 encoding='utf-8')

    prog = Progresser(len(questions['sentence']))
    # build the train data
    for i, qrow in enumerate(questions['sentence']):
        prog.count()
        snt = []
        for word in tokenise(qrow):
            if word not in stop_words:
                snt.append(word)
        snt = ' '.join(snt)
        for tpc in topics.cols:
            if topics[tpc][i] == '0':
                files[tpc + '-n'].write(snt + '\n')
            elif topics[tpc][i] == '1':
                files[tpc + '-p'].write(snt + '\n')
            else:
                print("wattt")

    for fl in files.values():
        fl.close()
Esempio n. 2
0
def create_w2v_vectors():
    # with open('./word2vec/IRBlog/w2v_per_300.pkl', 'rb') as infile:
    with open('./word2vec/Mixed/w2v_per.pkl', 'rb') as infile:
        w2v = pickle.load(infile)
    w2v_length = 100  # 300
    stop_words = set(
        pd.read_csv('./Primary_data/PersianStopWordList.txt', header=None)[0])
    questions = pd.read_csv('./Primary_data/result_filtered.csv',
                            delimiter=';')

    train = QuickDataFrame(['w' + str(i) for i in range(0, w2v_length)])

    prog = Progresser(questions.shape[0])
    # build the train data
    for i, qrow in questions.iterrows():
        prog.count()
        sum_array = np.zeros(w2v_length)
        number_of_words = 0

        for word in tokenise(qrow['sentence']):
            if word not in stop_words and word in w2v:
                number_of_words += 1
                sum_array += w2v[word]
        if i != len(train):
            print('wat?!!')
        train.append(list(sum_array / number_of_words))

    train.to_csv('./Primary_data/w2v-100_vector_Q.csv')
def find_frequent_words():
    id_mappings = QuickDataFrame.read_csv(
        './EurLex_data/eurlex_ID_mappings.csv', sep='\t')
    words = dict()

    prog = Progresser(len(id_mappings))
    for i in range(len(id_mappings)):
        prog.count()
        try:
            # read the file
            try:
                with open('./EurLex_data/lem_txt/' +
                          str(id_mappings['DocID'][i]) + '-lem.txt',
                          'r',
                          encoding="utf8") as infile:
                    doc_text = infile.read()
            except IOError as e:
                # print(e)
                continue
            # count the words
            for word in word_tokenize(doc_text):
                if word in words:
                    words[word] += 1
                else:
                    words[word] = 1
        except Exception as e:
            print(e)

    # remove bad words
    cleaner = re.compile('^(19\d\d)$|^(2\d\d\d)$|^((?!\d)\w)*$')
    filtered_words = dict()
    for word, freq in words.items():
        if cleaner.match(word):
            filtered_words[word] = freq

    sorted_words = sorted(filtered_words,
                          key=lambda x: filtered_words[x],
                          reverse=True)

    with open('./EurLex_data/words_frequency.csv', 'w',
              encoding="utf8") as outfile:
        for word in sorted_words:
            try:
                outfile.write(str(word) + ',' + str(words[word]) + '\n')
            except Exception as e:
                print(word, e)
                pass

    with open('./EurLex_data/1000words.csv', 'w', encoding="utf8") as outfile:
        for word in sorted_words[:1000]:
            outfile.write(str(word) + '\n')
def load_data(n_x=1000, n_y=160):
    data = QuickDataFrame.read_csv('./EurLex_data/eurlex_combined_vectors.csv')
    q_vector_length = 1000
    data.delete_column('doc_id')
    x_list = [data[col] for col in data.cols[:n_x]]
    x_array = np.array(x_list, dtype=int).transpose()

    y_list = []
    i = -1
    for col in data.cols[q_vector_length:]:
        i += 1
        if i < n_y:
            y_list.append(data[col])
    y_array = np.array(y_list, dtype=int).transpose()
    print('loaded data.')
    return x_array, y_array
def find_all_tags():
    subject_data = QuickDataFrame.read_csv(
        './EurLex_data/eurlex_id2class/id2class_eurlex_subject_matter.qrels',
        header=False,
        columns=['sub', 'doc_id', 'col2'],
        sep=' ')
    subjects = dict()
    for i in range(len(subject_data)):
        sub = subject_data['sub'][i]
        if sub not in subjects:
            subjects[sub] = 1
        else:
            subjects[sub] += 1

    sorted_tags = sorted(subjects, key=lambda x: subjects[x], reverse=True)
    with open('./EurLex_data/tags.csv', 'w') as outfile:
        outfile.write('term,freq\n')
        for tag in sorted_tags:
            outfile.write(tag + ',' + str(subjects[tag]) + '\n')
def build_w2v_vectors():
    with open('./word2vec/word2vec-En.pkl', 'rb') as infile:
        w2v = pickle.load(infile)

    w2v_length = 300
    stop_words = set()
    for w in stopwords.words('english'):
        stop_words.add(w)

    id_mappings = QuickDataFrame.read_csv(
        './EurLex_data/eurlex_ID_mappings.csv', sep='\t')

    # create DataFrame
    cols_list = ['doc_id'] + ['w' + str(i) for i in range(0, w2v_length)]
    train = QuickDataFrame(columns=cols_list)

    prog = Progresser(len(id_mappings))
    for i in range(len(id_mappings)):
        prog.count()
        # read the file
        try:
            with open('./EurLex_data/lem_txt/' + str(id_mappings['DocID'][i]) +
                      '-lem.txt',
                      'r',
                      encoding="utf8") as infile:
                doc_text = infile.read()
        except IOError:
            continue
        try:
            sum_array = np.zeros(w2v_length)
            number_of_words = 0

            for word in word_tokenize(doc_text):
                if word not in stop_words and word in w2v:
                    number_of_words += 1
                    sum_array += w2v[word]
            if number_of_words > 0:
                sum_array = sum_array / number_of_words

            train.append([id_mappings['DocID'][i]] + list(sum_array))

        except Exception as e:
            print(e)

    train.to_csv('./EurLex_data/w2v_vector_Q.csv')
def build_all_vectors():
    id_mappings = QuickDataFrame.read_csv(
        './EurLex_data/eurlex_ID_mappings.csv', sep='\t')
    subject_data = QuickDataFrame.read_csv(
        './EurLex_data/eurlex_id2class/id2class_eurlex_subject_matter.qrels',
        header=False,
        columns=['sub', 'doc_id', 'col2'],
        sep=' ')
    words_vector = QuickDataFrame.read_csv('./EurLex_data/1000words.csv',
                                           header=False,
                                           columns=['term'])
    topics = QuickDataFrame.read_csv('./EurLex_data/tags.csv')

    # train = QuickDataFrame.read_csv('./EurLex_data/w2v_vector_Q.csv')
    # train.set_index(train['doc_id'], unique=True)

    # create DataFrame
    cols_list = ['doc_id'] + list(words_vector['term'])
    train = QuickDataFrame(columns=cols_list)

    # filling word columns
    prog = Progresser(len(id_mappings))
    for i in range(len(id_mappings)):
        prog.count()
        try:
            # read the file
            try:
                with open('./EurLex_data/lem_txt/' +
                          str(id_mappings['DocID'][i]) + '-lem.txt',
                          'r',
                          encoding="utf8") as infile:
                    doc_text = infile.read()
            except IOError:
                continue

            # add a new row
            train.append(value=0)

            # complete the data in that row
            train['doc_id'][len(train) - 1] = id_mappings['DocID'][i]
            for word in word_tokenize(doc_text):
                if word in train.data:
                    train[word][len(train) - 1] = 1
        except Exception as e:
            print(e)

    # index by doc id
    train.set_index(train['doc_id'], unique=True)

    # rename word columns
    rename_dict = dict()
    index = 0
    for wrd in list(words_vector['term']):
        rename_dict[wrd] = 'wrd' + str(index)
        index += 1
    train.rename(columns=rename_dict)

    # add topic columns
    for col in list(topics['term']):
        train.add_column(name=col, value=0)

    # filling topic columns
    for i in range(len(subject_data)):
        try:
            sub = subject_data['sub'][i]
            doc_id = subject_data['doc_id'][i]
            train[sub, doc_id] = 1
        except Exception as e:
            print(e)

    # rename topic columns
    rename_dict = dict()
    index = 0
    for tpc in list(topics['term']):
        rename_dict[tpc] = 'tpc' + str(index)
        index += 1
    train.rename(columns=rename_dict)

    # write to file
    print('\nWriting to file...')
    # train.to_csv('./EurLex_data/eurlex_combined_vectors.csv')
    train.to_csv('./EurLex_data/eurlex_combined_vectors-w2v.csv')
def evaluate_model_cnn(x, y, learn_path, k=10):
    print(len(y), len(y[0]))
    # create a k fold with no unique classes
    count = 0
    while True:
        count += 1
        # print(count, 'Finding a proper KF...')
        kf = list(
            KFold(n_splits=k, shuffle=True,
                  random_state=randint(0, 100000)).split(x))
        good_folds = True
        for train_index, test_index in kf:
            for i in range(len(y[0])):
                if len(np.unique(
                        y[train_index,
                          i])) < 2:  # or len(np.unique(y[test_index, i])) < 2:
                    # print(y[train_index, i],np.unique(y[train_index, i]))
                    print(i)
                    good_folds = False
                    break
            if not good_folds:
                break
        if good_folds:
            break
    print('Found a good KF in', count, 'try!')

    with open(learn_path + 'topic_classifier-folds.pkl', 'wb') as out_file:
        pickle.dump(kf, out_file)
    fold_num = 0

    stats = QuickDataFrame([
        'Jaccard (normalised)', 'Accuracy (normalised)', 'Accuracy',
        'F1_score (micro averaged)', 'F1_score (macro averaged by labels)',
        'F1_score (averaged by samples)', 'Hamming loss', 'Label Ranking loss:'
    ])

    txt_cnns = [TextCNN() for _ in range(y.shape[1])]
    prog = Progresser(k)
    for train_index, test_index in kf:
        # print(train_index, test_index)
        print('___________________________________________________')
        x_train, x_test = x[train_index], x[test_index]
        y_train, y_test = y[train_index], y[test_index]

        # cls = SVC(kernel='linear')
        # cls = SVC(kernel='poly', probability=True, tol=1e-5)
        # cls = GaussianNB()
        # cls = RandomForestClassifier(max_features='auto', random_state=1)

        # topic_classifier = BinaryRelevance(classifier=cls, require_dense=[True, True])

        # try:
        # topic_classifier.fit(x_train, y_train)
        for i in range(y.shape[1]):
            txt_cnns[i].train_cnn(train_index, i)
        # except Exception as e:
        #     print('\nfit error!:', e)
        #     continue

        # with open(learn_path + 'topic_classifier-SVC' + str(fold_num) + '.pkl', 'wb') as out_file:
        #     pickle.dump(topic_classifier, out_file)

        # try:
        predictions = np.zeros((len(x_test), y.shape[1]))
        for i in range(y.shape[1]):
            predictions[:, i] = np.array(txt_cnns[i].predict_text(test_index))

        s = [
            jaccard_score(y_test, predictions, average='micro'),
            accuracy_score(y_test, predictions, normalize=True),
            accuracy_score(y_test, predictions, normalize=False),
            f1_score(y_test, predictions, average='micro'),
            f1_score(y_test, predictions, average='macro'),
            f1_score(y_test, predictions, average='samples'),
            hamming_loss(y_test, predictions),
            label_ranking_loss(y_test, predictions)
        ]

        stats.append(s)
        print(stats[stats.length - 1])
        # except Exception as e:
        #     print('Eval error!:', e)

        fold_num += 1
        prog.count()

    for col in stats.cols:
        print(col, np.mean(stats[col]))