def _get_csv_file_content(csv_file):
"""
returns appropriate csv file content based on input and output is
compatible with python versions
"""
if (not isinstance(csv_file, str)) and six.PY3:
content = csv_file.read()
else:
content = csv_file
if isinstance(content, bytes):
csv_content = content.decode('utf-8')
else:
csv_content = content
if six.PY3:
return csv_content
else:
return UniversalNewlineIterator(csv_content)
def test_unicode_data(self, buffer_size):
assert [
thing.decode('utf-8')
for thing in UniversalNewlineIterator(StringIO('héllø wo®ld'),
buffer_size=buffer_size)
] == ['héllø wo®ld']
def test_empty_file(self, buffer_size):
assert [
thing.decode('utf-8')
for thing in UniversalNewlineIterator(StringIO(''),
buffer_size=buffer_size)
] == []
def test_only_one_line_no_trailing_newline(self, buffer_size):
assert [
thing.decode('utf-8')
for thing in UniversalNewlineIterator(StringIO('foo'),
buffer_size=buffer_size)
] == ['foo']
def test_carriage_returns_and_line_feeds(self, buffer_size):
assert [
thing.decode('utf-8')
for thing in UniversalNewlineIterator(StringIO('foo\r\nbar\r\n'),
buffer_size=buffer_size)
] == ['foo\n', 'bar\n']
def test_unicode_data(self, buffer_size):
self.assertEqual([thing.decode('utf-8') if six.PY3 else thing for thing in
UniversalNewlineIterator(StringIO(u'héllø wo®ld'), buffer_size=buffer_size)], [u'héllø wo®ld']) # lint-amnesty, pylint: disable=line-too-long
def test_empty_file(self, buffer_size):
self.assertEqual(
[thing.decode('utf-8') for thing in UniversalNewlineIterator(StringIO(u''), buffer_size=buffer_size)],
[]
)
def test_only_one_line(self, buffer_size):
self.assertEqual(
[thing.decode('utf-8') for thing in UniversalNewlineIterator(StringIO(u'foo\n'), buffer_size=buffer_size)],
['foo\n']
)
def test_no_trailing_newline(self, buffer_size):
self.assertEqual(
[thing.decode('utf-8') for thing in UniversalNewlineIterator(StringIO(u'foo\nbar'), buffer_size=buffer_size)], # lint-amnesty, pylint: disable=line-too-long
['foo\n', 'bar']
)
def test_carriage_returns_and_line_feeds(self, buffer_size):
self.assertEqual(
[thing.decode('utf-8') for thing in UniversalNewlineIterator(StringIO(u'foo\r\nbar\r\n'), buffer_size=buffer_size)], # lint-amnesty, pylint: disable=line-too-long
['foo\n', 'bar\n']
)
def test_no_trailing_newline(self, buffer_size):
assert [thing.decode('utf-8') for thing in
UniversalNewlineIterator(StringIO(u'foo\nbar'), buffer_size=buffer_size)] == ['foo\n', 'bar']
def test_carriage_returns(self, buffer_size):
self.assertEqual([
thing.decode('utf-8')
for thing in UniversalNewlineIterator(StringIO(u'foo\rbar\r'),
buffer_size=buffer_size)
], ['foo\n', 'bar\n'])
