Ejemplo n.º 1
0
Archivo: rss.py Proyecto: stes/nbot
def read_data(feed, download=False):
    # download the feed
    log('fetch feed: %s' % feed.title)
    content = feed.description
    if download:
        content = htmlparser.fetch_content(feed.link)
    doc = Document(preprocess(content))
    log('content:\n%s' % content)
    return doc
Ejemplo n.º 2
0
Archivo: recsys.py Proyecto: stes/nbot
def gen_feature_vector(mask, text):
    '''
    Generates a feature vector by applying the given mask to the specified
    document.
    The feature vector is generated by mapping the words in the mask list to
    the corresponding number of occurrences of these words in the document.
    Therefore, the length of the mask specifies the length of the output list.
    See also: 'Bag-of-words model'
    
    @param mask: a list of words that should be used as the mask
    @param text: the text for which the feature vector should be generated
    
    @return: a list of integers representing occurrences of words in the
    document
    '''
    processed = preprocess(text)
    vlist = VocabList()
    vlist.expand_with(processed)
    fvector = []
    for word in mask:
        fvector.append(vlist.quantity_of(word) / float(vlist.get_total_word_count()))
    return fvector