Пример #1
0
    def test_escapechar(self):
        data = '\\"hello world"\n' '\\"hello world"\n' '\\"hello world"'

        reader = TextReader(StringIO(data), delimiter=",", header=None, escapechar="\\")
        result = reader.read()
        expected = {0: ['"hello world"'] * 3}
        assert_array_dicts_equal(result, expected)
Пример #2
0
 def test_file_handle(self):
     try:
         f = open(self.csv1, 'rb')
         reader = TextReader(f)
         result = reader.read()  # noqa
     finally:
         f.close()
Пример #3
0
 def test_file_handle_mmap(self):
     try:
         f = open(self.csv1, 'rb')
         reader = TextReader(f, memory_map=True, header=None)
         reader.read()
     finally:
         f.close()
Пример #4
0
    def test_header_not_enough_lines_as_recarray(self):

        if compat.is_platform_windows():
            raise nose.SkipTest(
                "segfaults on win-64, only when all tests are run")

        data = ('skip this\n'
                'skip this\n'
                'a,b,c\n'
                '1,2,3\n'
                '4,5,6')

        reader = TextReader(StringIO(data), delimiter=',', header=2,
                            as_recarray=True)
        header = reader.header
        expected = [['a', 'b', 'c']]
        self.assertEqual(header, expected)

        recs = reader.read()
        expected = {'a': [1, 4], 'b': [2, 5], 'c': [3, 6]}
        assert_array_dicts_equal(expected, recs)

        # not enough rows
        self.assertRaises(parser.CParserError, TextReader, StringIO(data),
                          delimiter=',', header=5, as_recarray=True)
Пример #5
0
    def test_parse_booleans(self):
        data = 'True\nFalse\nTrue\nTrue'

        reader = TextReader(StringIO(data), header=None)
        result = reader.read()

        self.assertEqual(result[0].dtype, np.bool_)
Пример #6
0
    def test_embedded_newline(self):
        data = 'a\n"hello\nthere"\nthis'

        reader = TextReader(StringIO(data), header=None)
        result = reader.read()

        expected = ["a", "hello\nthere", "this"]
        self.assert_(np.array_equal(result[0], expected))
Пример #7
0
    def test_delimit_whitespace(self):
        data = 'a  b\na\t\t "b"\n"a"\t \t b'

        reader = TextReader(StringIO(data), delim_whitespace=True, header=None)
        result = reader.read()

        self.assert_(np.array_equal(result[0], ["a", "a", "a"]))
        self.assert_(np.array_equal(result[1], ["b", "b", "b"]))
Пример #8
0
    def test_skipinitialspace(self):
        data = "a,   b\n" "a,   b\n" "a,   b\n" "a,   b"

        reader = TextReader(StringIO(data), skipinitialspace=True, header=None)
        result = reader.read()

        self.assert_(np.array_equal(result[0], ["a", "a", "a", "a"]))
        self.assert_(np.array_equal(result[1], ["b", "b", "b", "b"]))
Пример #9
0
    def test_euro_decimal(self):
        data = "12345,67\n345,678"

        reader = TextReader(StringIO(data), delimiter=":", decimal=",", header=None)
        result = reader.read()

        expected = [12345.67, 345.678]
        tm.assert_almost_equal(result[0], expected)
Пример #10
0
    def test_embedded_newline(self):
        data = 'a\n"hello\nthere"\nthis'

        reader = TextReader(StringIO(data), header=None)
        result = reader.read()

        expected = np.array(['a', 'hello\nthere', 'this'], dtype=np.object_)
        self.assert_numpy_array_equal(result[0], expected)
Пример #11
0
    def test_integer_thousands(self):
        data = "123,456\n12,500"

        reader = TextReader(StringIO(data), delimiter=":", thousands=",", header=None)
        result = reader.read()

        expected = [123456, 12500]
        tm.assert_almost_equal(result[0], expected)
Пример #12
0
    def test_delimit_whitespace(self):
        data = 'a  b\na\t\t "b"\n"a"\t \t b'

        reader = TextReader(StringIO(data), delim_whitespace=True,
                            header=None)
        result = reader.read()

        self.assert_numpy_array_equal(result[0], ['a', 'a', 'a'])
        self.assert_numpy_array_equal(result[1], ['b', 'b', 'b'])
Пример #13
0
    def test_euro_decimal(self):
        data = '12345,67\n345,678'

        reader = TextReader(StringIO(data), delimiter=':',
                            decimal=',', header=None)
        result = reader.read()

        expected = np.array([12345.67, 345.678])
        tm.assert_almost_equal(result[0], expected)
Пример #14
0
    def test_integer_thousands(self):
        data = '123,456\n12,500'

        reader = TextReader(StringIO(data), delimiter=':',
                            thousands=',', header=None)
        result = reader.read()

        expected = np.array([123456, 12500], dtype=np.int64)
        tm.assert_almost_equal(result[0], expected)
Пример #15
0
    def test_skipinitialspace(self):
        data = ('a,   b\n'
                'a,   b\n'
                'a,   b\n'
                'a,   b')

        reader = TextReader(StringIO(data), skipinitialspace=True,
                            header=None)
        result = reader.read()

        self.assert_numpy_array_equal(result[0], ['a', 'a', 'a', 'a'])
        self.assert_numpy_array_equal(result[1], ['b', 'b', 'b', 'b'])
Пример #16
0
    def test_header_not_enough_lines(self):
        data = "skip this\n" "skip this\n" "a,b,c\n" "1,2,3\n" "4,5,6"

        reader = TextReader(StringIO(data), delimiter=",", header=2, as_recarray=True)
        header = reader.header
        expected = [["a", "b", "c"]]
        self.assertEquals(header, expected)

        recs = reader.read()
        expected = {"a": [1, 4], "b": [2, 5], "c": [3, 6]}
        assert_array_dicts_equal(expected, recs)

        # not enough rows
        self.assertRaises(parser.CParserError, TextReader, StringIO(data), delimiter=",", header=5, as_recarray=True)
Пример #17
0
    def test_header_not_enough_lines(self):
        data = ('skip this\n'
                'skip this\n'
                'a,b,c\n'
                '1,2,3\n'
                '4,5,6')

        reader = TextReader(StringIO(data), delimiter=',', header=2)
        header = reader.header
        expected = [['a', 'b', 'c']]
        self.assertEqual(header, expected)

        recs = reader.read()
        expected = {0: [1, 4], 1: [2, 5], 2: [3, 6]}
        assert_array_dicts_equal(expected, recs)

        # not enough rows
        self.assertRaises(parser.CParserError, TextReader, StringIO(data),
                          delimiter=',', header=5, as_recarray=True)
Пример #18
0
    def test_skip_bad_lines(self):
        # too many lines, see #2430 for why
        data = "a:b:c\n" "d:e:f\n" "g:h:i\n" "j:k:l:m\n" "l:m:n\n" "o:p:q:r"

        reader = TextReader(StringIO(data), delimiter=":", header=None)
        self.assertRaises(parser.CParserError, reader.read)

        reader = TextReader(StringIO(data), delimiter=":", header=None, error_bad_lines=False, warn_bad_lines=False)
        result = reader.read()
        expected = {0: ["a", "d", "g", "l"], 1: ["b", "e", "h", "m"], 2: ["c", "f", "i", "n"]}
        assert_array_dicts_equal(result, expected)

        stderr = sys.stderr
        sys.stderr = StringIO()
        try:
            reader = TextReader(StringIO(data), delimiter=":", header=None, error_bad_lines=False, warn_bad_lines=True)
            reader.read()
            val = sys.stderr.getvalue()
            self.assertTrue("Skipping line 4" in val)
            self.assertTrue("Skipping line 6" in val)
        finally:
            sys.stderr = stderr
Пример #19
0
    def test_skip_bad_lines(self):
        # too many lines, see #2430 for why
        data = ('a:b:c\n'
                'd:e:f\n'
                'g:h:i\n'
                'j:k:l:m\n'
                'l:m:n\n'
                'o:p:q:r')

        reader = TextReader(StringIO(data), delimiter=':',
                            header=None)
        self.assertRaises(parser.CParserError, reader.read)

        reader = TextReader(StringIO(data), delimiter=':',
                            header=None,
                            error_bad_lines=False,
                            warn_bad_lines=False)
        result = reader.read()
        expected = {0: ['a', 'd', 'g', 'l'],
                    1: ['b', 'e', 'h', 'm'],
                    2: ['c', 'f', 'i', 'n']}
        assert_array_dicts_equal(result, expected)

        stderr = sys.stderr
        sys.stderr = StringIO()
        try:
            reader = TextReader(StringIO(data), delimiter=':',
                                header=None,
                                error_bad_lines=False,
                                warn_bad_lines=True)
            reader.read()
            val = sys.stderr.getvalue()
            self.assertTrue('Skipping line 4' in val)
            self.assertTrue('Skipping line 6' in val)
        finally:
            sys.stderr = stderr
Пример #20
0
 def _test(text, **kwargs):
     nice_text = text.replace('\r', '\r\n')
     result = TextReader(StringIO(text), **kwargs).read()
     expected = TextReader(StringIO(nice_text), **kwargs).read()
     assert_array_dicts_equal(result, expected)
Пример #21
0
 def test_string_factorize(self):
     # should this be optional?
     data = 'a\nb\na\nb\na'
     reader = TextReader(StringIO(data), header=None)
     result = reader.read()
     self.assertEqual(len(set(map(id, result[0]))), 2)
Пример #22
0
 def test_StringIO(self):
     text = open(self.csv1, 'rb').read()
     src = BytesIO(text)
     reader = TextReader(src, header=None)
     reader.read()
Пример #23
0
 def test_string_filename(self):
     reader = TextReader(self.csv1, header=None)
     reader.read()
Пример #24
0
 def test_StringIO(self):
     text = open(self.csv1, 'rb').read()
     src = BytesIO(text)
     reader = TextReader(src, header=None)
     result = reader.read()
Пример #25
0
 def test_string_filename(self):
     reader = TextReader(self.csv1, header=None)
     reader.read()
Пример #26
0
 def test_StringIO(self):
     with open(self.csv1, 'rb') as f:
         text = f.read()
     src = BytesIO(text)
     reader = TextReader(src, header=None)
     reader.read()
Пример #27
0
 def test_string_factorize(self):
     # should this be optional?
     data = 'a\nb\na\nb\na'
     reader = TextReader(StringIO(data), header=None)
     result = reader.read()
     self.assertEqual(len(set(map(id, result[0]))), 2)
Пример #28
0
 def _make_reader(**kwds):
     return TextReader(StringIO(data),
                       delimiter=',',
                       header=None,
                       **kwds)