def __get_tokens(self, row):
row_tokenizer = RowTokenizer(row, self.config)
line = row_tokenizer.next()
while line:
#print "RETURN", line[0:100]
yield line
line = row_tokenizer.next()
def __get_tokens(self, row):
"""Row should be a single string"""
row_tokenizer = RowTokenizer(row, self.config)
line = row_tokenizer.next()
while line:
# print("RETURN line", line[0:100])
yield line
line = row_tokenizer.next()
def __get_tokens(self, row):
"""Row should be a single string"""
row_tokenizer = RowTokenizer(row, self.config)
line = row_tokenizer.next()
while line:
# print("RETURN line", line[0:100])
yield line
line = row_tokenizer.next()