words = []
            # combine wordlists of one doc in order to get a list of words
            for wordlist in doc:
                words += wordlist
            words = Preprocessing.Wordlist_cleaner(words, remove_puncts=True)

            # words = ['i','am','good'] or words = []
            Use_this_words = False
            for word in words:
                if word in index2word_set:
                    Use_this_words = True
                    break
            if Use_this_words == False:
                empty_docs.add(j)

            X = BuildFeature.getAvgFeatureVecs([words], model)
            y = np.array([emotion])
            FeatureVec = {'X': X, 'y': y}
            new_docs_dict[j] = FeatureVec
            if (j + 1) % 1000. == 0.:
                print ">>  %s - %d/%d doc finished! " % (emotion, j + 1,
                                                         len(docs))
        empty_docs_index[emotion] = empty_docs
        pickle.dump(
            new_docs_dict,
            open(save_path + '/' + feature_name + '.' + emotion + '.Xy.pkl',
                 'wb'))
        print ">> %s. %s emotion finishing!" % (i + 1, emotion)
        del new_docs_dict
        del empty_docs
    pickle.dump(empty_docs_index,
                avg_number_of_sentences = n_sentences / number_of_parts
                number_of_sentences_remain = n_sentences % number_of_parts
                start_row = 0
                for s in xrange(number_of_parts):
                    if number_of_sentences_remain <= 0:
                        Part_of_clean_doc = clean_doc[start_row:start_row +
                                                      avg_number_of_sentences]
                    else:
                        Part_of_clean_doc = clean_doc[start_row:start_row +
                                                      avg_number_of_sentences +
                                                      1]
                    start_row = start_row + len(Part_of_clean_doc)
                    Part_of_clean_doc = sum(Part_of_clean_doc, [])
                    new_doc.append(Part_of_clean_doc)
                    number_of_sentences_remain = number_of_sentences_remain - 1

            DocVec = BuildFeature.getAvgFeatureVecs(new_doc, model)
            DocVec = np.reshape(DocVec, np.product(DocVec.shape))
            X[idx] = DocVec
            idx = idx + 1

            if (j + 1) % 100. == 0.:
                print ">>  %s - %d/%d doc finished! " % (emotion, j + 1,
                                                         len(docs))
            del new_doc, clean_doc, DocVec

        label = np.array([emotion] * len(docs))
        y = np.concatenate((y, label), axis=1)
        del docs
        print ">> %s. %s emotion finishing!" % (i + 1, emotion)
    np.savez_compressed(save_path, X=X, y=y)
        print 'Start to load ' + emotion + '_wordlists.pkl'
        docs = pickle.load(
            open(load_path + '/' + emotion + '_wordlists.pkl', "rb"))
        # docs = docs[0:5]
        for j, doc in enumerate(docs):
            new_doc_keyword = []
            new_doc = sum(doc, [])
            for word in new_doc:
                if word in basic_keyword_list and word in index2word_set:
                    new_doc_keyword.append(word)

            # print new_doc_keyword
            if len(new_doc_keyword) == 0:
                empty_docs.add((emotion, j))
                new_docs.append(new_doc)
            else:
                new_docs.append(new_doc_keyword)

            if (j + 1) % 100. == 0.:
                print ">>  %s - %d/%d doc finished! " % (emotion, j + 1,
                                                         len(docs))

        label = np.array([emotion] * len(docs))
        y = np.concatenate((y, label), axis=1)
        print ">> %s. %s emotion finishing!" % (i + 1, emotion)

    print empty_docs

    X = BuildFeature.getAvgFeatureVecs(new_docs, model)
    np.savez_compressed(save_path, X=X, y=y)
                Use_this_wordlist = False
                wordlist = Preprocessing.Wordlist_cleaner(wordlist,remove_puncts=True)

                if len(wordlist) > 0 :
                    for word in wordlist:
                        if word in index2word_set:
                            Use_this_wordlist = True
                            break
                    if Use_this_wordlist == True:
                        new_doc.append(wordlist)

            if len(new_doc) == 0:
                empty_docs.add(j)
                new_doc = [new_doc]

            X = BuildFeature.getAvgFeatureVecs(new_doc, model)
            y = np.array([emotion]*len(new_doc))
            FeatureVec = {'X': X, 'y': y}
            new_docs_dict[j] = FeatureVec
            if (j+1)%100. == 0.:
                print ">>  %s - %d/%d doc finished! " % (emotion,j+1,len(docs))
            del new_doc
            
        empty_docs_index[emotion] = empty_docs
        pickle.dump(new_docs_dict, open(save_path+'/'+feature_name+'.'+emotion+'.Xy.pkl', 'wb'))
        print ">> %s. %s emotion finishing!" % (i+1,emotion)
        del new_docs_dict
        # print emotion," ",empty_docs
        del empty_docs
    pickle.dump(empty_docs_index, open(save_path+'/empty_docs_index.pkl', 'wb'))
                    for word in wordlist:
                        # if word in basic_keyword_list and word in index2word_set:
                        if word in extend_keyword_list and word in index2word_set:
                            keyword_inside = True
                            break
                    if Use_this_wordlist == True:
                        new_doc.append(wordlist)
                    if keyword_inside == True:
                        new_doc_with_keywords.append(wordlist)

            if len(new_doc) == 0:
                empty_docs.add(j)
                new_doc = [new_doc]

            if len(new_doc_with_keywords) == 0:
                SentenceVecs = BuildFeature.getAvgFeatureVecs(new_doc, model)
            else:
                SentenceVecs = BuildFeature.getAvgFeatureVecs(new_doc_with_keywords, model)
            
            n_sentences = SentenceVecs.shape[0]
            DocVec = np.zeros((n_features,),dtype="float32")

            # print 'SentenceVecs : ', SentenceVecs.shape
            for row in xrange(n_sentences):
                DocVec = np.add(DocVec,SentenceVecs[row])
            DocVec = np.divide(DocVec,n_sentences)
            del SentenceVecs

            X[idx] = DocVec
            idx = idx + 1
Beispiel #6
0
                    for word in wordlist:
                        # if word in basic_keyword_list and word in index2word_set:
                        if word in extend_keyword_list and word in index2word_set:
                            keyword_inside = True
                            break
                    if Use_this_wordlist == True:
                        new_doc.append(wordlist)
                    if keyword_inside == True:
                        new_doc_with_keywords.append(wordlist)

            if len(new_doc) == 0:
                empty_docs.add(j)
                new_doc = [new_doc]

            if len(new_doc_with_keywords) == 0:
                SentenceVecs = BuildFeature.getAvgFeatureVecs(new_doc, model)
            else:
                SentenceVecs = BuildFeature.getAvgFeatureVecs(
                    new_doc_with_keywords, model)

            n_sentences = SentenceVecs.shape[0]
            DocVec = np.zeros((n_features, ), dtype="float32")

            # print 'SentenceVecs : ', SentenceVecs.shape
            for row in xrange(n_sentences):
                DocVec = np.add(DocVec, SentenceVecs[row])
            DocVec = np.divide(DocVec, n_sentences)
            del SentenceVecs

            X[idx] = DocVec
            idx = idx + 1