def read_data(feed, download=False):
# download the feed
log('fetch feed: %s' % feed.title)
content = feed.description
if download:
content = htmlparser.fetch_content(feed.link)
doc = Document(preprocess(content))
log('content:\n%s' % content)
return doc
def gen_feature_vector(mask, text):
'''
Generates a feature vector by applying the given mask to the specified
document.
The feature vector is generated by mapping the words in the mask list to
the corresponding number of occurrences of these words in the document.
Therefore, the length of the mask specifies the length of the output list.
See also: 'Bag-of-words model'
@param mask: a list of words that should be used as the mask
@param text: the text for which the feature vector should be generated
@return: a list of integers representing occurrences of words in the
document
'''
processed = preprocess(text)
vlist = VocabList()
vlist.expand_with(processed)
fvector = []
for word in mask:
fvector.append(vlist.quantity_of(word) / float(vlist.get_total_word_count()))
return fvector