def train_and_save_model(self, filename, spotter):
afinn_filename = '../sellibrary/resources/AFINN-111.txt'
sentiment_processor = SentimentProcessor()
self.train_model_using_dexter_dataset(sentiment_processor, spotter,
afinn_filename)
sentiment_processor.save_model(filename)
return sentiment_processor
def unit_test_2():
afinn_filename = '../resources/AFINN-111.txt'
sp = SentimentProcessor(afinn_filename)
x = sp.get_ngram('a b c d e f g h i j k', 3, 8, 9)
assert (x == 'b c d e f g h')
x = sp.get_ngram('a b c d e f g h i j k', 4, 8, 9)
assert (x == 'a b c d e f g h i')
x = sp.get_ngram('a b c d e f g h i j k', 6, 8, 9)
assert (x == 'a b c d e f g h i j k')
s = '%d %d 0 0 [ %f ]' % (docid, entity_id, sent)
self.logger.info(s)
file_contents = file_contents + s + '\n'
file = open(output_filename, "w")
file.write(file_contents)
file.close()
self.logger.info('processing complete')
# def train_and_save_model(self, filename):
# spotter = SpotlightCachingSpotter(False)
# afinn_filename = '../sellibrary/resources/AFINN-111.txt'
# sentiment_processor = SentimentProcessor()
# self.train_model_using_dexter_dataset(sentiment_processor, spotter, afinn_filename)
# sentiment_processor.save_model(filename)
# return sentiment_processor
if __name__ == "__main__":
filename = FileLocations.get_dropbox_intermediate_path(
) + 'sentiment.pickle'
dexter_feeder = DexterFeeder()
spotter = SpotlightCachingSpotter(False)
sentiment_processor = SentimentProcessor()
sentiment_processor.load_model(filename)
dexter_feeder.dexter_dataset_sentiment(sentiment_processor, spotter,
'/tmp/sentiment_output.txt')
def unit_test1():
afinn_filename = '../resources/AFINN-111.txt'
max_diff = 0.0001
sp = SentimentProcessor(afinn_filename)
sp.cal_term_weight_on_full_corpus(
['abhor abandoned archaeological', ' happy apple archaeological '])
sp.__calc_term_total_sent()
print('happy has a doc normalised value of:',
sp.get_doc_sentiment('happy'))
print('archaeological has a doc normalised value of:',
sp.get_doc_sentiment('archaeological'))
assert (sp.get_doc_sentiment('archaeolog') < 0
) # with 2 neg, and 1 +ve word
assert (abs(sp.get_doc_sentiment('happy') - 0.8571428571428571) < max_diff)
assert (sp.get_doc_sentiment('archaeolog') < 0
) # with 2 neg, and 1 +ve word