Example #1
0
def do_lda(num_topics, passes, iterations, chunksize, tfidf, wiki_path=None):
    model_name = 'tweets_'

    logging.info('Loading user dictionary...')
    dictionary = corpora.Dictionary.load('data/dictionary/tweets.dict')
    corpus = dictionary.corpus

    if tfidf is True:
        logging.info('Computing TF-IDF...')
        tfidf_model = TfidfModel(corpus, normalize=False)
        corpus = tfidf_model[corpus]
        logging.info('Transforming the corpus...')
        corpus = [tfidf_model[corpus] for corpus in corpus]
        model_name += 'tfidf_'

    if wiki_path is not None:
        model_name += 'wiki_'

    logging.info('Performing LDA on user corpus...')
    model, vectors, ids = perform_lda(dictionary=dictionary, corpus=corpus, num_topics=num_topics, passes=passes,
                                      iterations=iterations, chunksize=chunksize, wiki_path=wiki_path)
    model.print_topics(topics=num_topics, topn=10)
    model.save('data/model/' + model_name + str(num_topics) + '.lda')
    enpickle(vectors, 'data/vector/' + model_name + str(num_topics) + '.pkl')
    enpickle(ids, 'data/vector/ids.pkl')
def crawl_wiki(model_path):
    engine = Wikipedia(license=None, throttle=1.0, language='en')
    wikis = {}
    keywords = get_keywords(model_path=model_path, threshold=0.001)
    for keyword in keywords:
        stop = False
        while stop is False:
            try:
                article = engine.search(query=keyword)
            except Exception as e:
                print str(e)
                article = None

            if type(article) is pattern.web.WikipediaArticle:
                if article.disambiguation is False:
                    print '\nretrieving', keyword, '...',
                    wikis[keyword] = {}
                    wikis[keyword]['keyword'] = keyword
                    wikis[keyword]['text'] = article.plaintext()
                    stop = True
                else:
                    print '\n[', keyword, '] leads to disambiguation page!',
                    stop = True

                    if '-' in keyword:
                        keyword = re.sub('-', ' ', keyword)  # convert hyphen into white space
                        stop = False
                    if keyword.islower() and len(keyword) <= 5:
                        keyword = keyword.upper()
                        stop = False
            else:
                print '\n[', keyword, '] doesn\'t exist on wikipedia!',
                stop = True

                if '-' in keyword:
                    keyword = re.sub('-', ' ', keyword)  # convert hyphen into white space
                    stop = False
                if keyword.islower() and len(keyword) <= 5:
                    keyword = keyword.upper()
                    stop = False

    enpickle(wikis, 'data/others/wikis.pkl')
    print '\n'
    return wikis
    count = 0
    doc_num = len(wikis)
    new_wikis = []
    keywords = []
    for keyword, wiki in wikis.items():
        count += 1

        print '\r', count, '/', doc_num,
        text = wiki['text']
        cleaned = clean_text(text)  # delete irrelevant characters

        wiki = []
        tokens = lemmatize(content=cleaned, allowed_tags=allowed_pos)  # lemmatize
        for token in tokens:
            word, pos = token.split('/')
            wiki.append(word)

        # convert compound word into one token
        wiki = convert_compound(wiki)

        # filter stop words, long words, and non-english words
        wiki = [w for w in wiki if not w in stopwords and 2 <= len(w) <= 15 and w.islower()]  # FIXME: this allows non-english characters to be stored

        new_wikis.append(wiki)
        keywords.append(keyword)

    print '\n'

    logging.info('Saving wiki corpus...')
    enpickle(new_wikis, 'data/processed/wikis.pkl')
Example #4
0
            continue
        if not text_index.has_key(text):
            text_index[text] = ''
            tweets[id] = text

    # other english tweets
    other_file = open('data/original/agreegatedEnglishTweets.csv', 'rb')
    other_csv = csv.reader(other_file)
    for row in other_csv:
        id, text = row
        id = 'other_' + id
        if len(text) < 10:
            continue
        if not text_index.has_key(text):
            text_index[text] = ''
            tweets[id] = text

    # additional tweets
    other_file = open('data/original/additional.csv', 'rb')
    other_csv = csv.reader(other_file)
    for row in other_csv:
        id, text = row
        id = 'additional_' + id
        if len(text) < 10:
            continue
        if not text_index.has_key(text):
            text_index[text] = ''
            tweets[id] = text

    enpickle(tweets, 'data/processed/tweets.pkl')