class ReviewCorpus(object):
    def __init__(self, filename, dictionary):
        self.filename = filename
        self.dictionary = dictionary
        self.tokenizer = SimpleTokenizer()

    def __iter__(self):
        with open(self.filename) as f:
            for line in f:
                review = json.loads(line)
                tokens = self.tokenizer.tokenize(review)
                yield self.dictionary.doc2bow(tokens)
def load_reviews(filename):
    tokeniser = SimpleTokenizer()
    with open(filename, 'r') as f:
        for line in f:
            review = json.loads(line)
            yield tokeniser.tokenize(review)
 def __init__(self, filename, dictionary):
     self.filename = filename
     self.dictionary = dictionary
     self.tokenizer = SimpleTokenizer()