def __get_tokens(self, row):
        row_tokenizer = RowTokenizer(row, self.config)

        line = row_tokenizer.next()
        while line:
            #print "RETURN", line[0:100]
            yield line
            line = row_tokenizer.next()
Exemple #2
0
    def __get_tokens(self, row):
        """Row should be a single string"""
        row_tokenizer = RowTokenizer(row, self.config)

        line = row_tokenizer.next()
        while line:
            # print("RETURN line", line[0:100])
            yield line
            line = row_tokenizer.next()
    def __get_tokens(self, row):
        """Row should be a single string"""
        row_tokenizer = RowTokenizer(row, self.config)

        line = row_tokenizer.next()
        while line:
            # print("RETURN line", line[0:100])
            yield line
            line = row_tokenizer.next()