def test_auto_type_inference(self):
data = StringIO(
'0,1,2,3,4\n5.5,6,7,8,9\n10,11,12,13,14a\n15,16,xxx,18,19')
adapter = textadapter.text_adapter(data,
field_names=False,
infer_types=True)
array = adapter.to_array()
self.assert_equality(array.dtype.fields['f0'][0], np.dtype('float64'))
self.assert_equality(array.dtype.fields['f1'][0], np.dtype('uint64'))
self.assert_equality(array.dtype.fields['f2'][0], np.dtype('O'))
self.assert_equality(array.dtype.fields['f3'][0], np.dtype('uint64'))
self.assert_equality(array.dtype.fields['f4'][0], np.dtype('O'))
data = StringIO(
'0,1,2,3,4\n5.5,6,7,8,9\n10,11,12,13,14a\n15,16,xxx,18,19')
adapter = textadapter.text_adapter(data,
field_names=False,
infer_types=True)
self.assert_equality(adapter[0].dtype.fields['f0'][0],
np.dtype('uint64'))
self.assert_equality(adapter[1:3].dtype.fields['f0'][0],
np.dtype('float64'))
self.assert_equality(adapter[3].dtype.fields['f4'][0],
np.dtype('uint64'))
self.assert_equality(adapter[:].dtype.fields['f3'][0],
np.dtype('uint64'))
self.assert_equality(adapter[-1].dtype.fields['f2'][0], np.dtype('O'))
self.assert_equality(adapter[2].dtype.fields['f4'][0], np.dtype('O'))
def test_field_names(self):
# Test for ignoring of extra fields
data = StringIO('f0,f1\n0,1,2\n3,4,5')
adapter = textadapter.text_adapter(data,
'csv',
delimiter=',',
field_names=True)
array = adapter.to_array()
self.assert_equality(array.dtype.names, ('f0', 'f1'))
self.assert_equality(array[0].item(), (0, 1))
self.assert_equality(array[1].item(), (3, 4))
# Test for duplicate field names
data = StringIO('f0,field,field\n0,1,2\n3,4,5')
adapter = textadapter.text_adapter(data,
'csv',
delimiter=',',
field_names=True,
infer_types=False)
adapter.set_field_types({0: 'u4', 1: 'u4', 2: 'u4'})
array = adapter.to_array()
self.assert_equality(array.dtype.names, ('f0', 'field', 'field1'))
# Test for field names list
data = StringIO('0,1,2\n3,4,5')
adapter = textadapter.text_adapter(data,
field_names=['a', 'b', 'c'],
infer_types=False)
adapter.field_types = {0: 'u4', 1: 'u4', 2: 'u4'}
array = adapter[:]
self.assertTrue(array.dtype.names == ('a', 'b', 'c'))
assert_array_equal(
array,
np.array([(0, 1, 2), (3, 4, 5)],
dtype=[('a', 'u4'), ('b', 'u4'), ('c', 'u4')]))
def test_delimiter(self):
data = StringIO('1,2,3\n')
adapter = textadapter.text_adapter(data, field_names=False)
self.assert_equality(adapter[0].item(), (1, 2, 3))
data = StringIO('1 2 3\n')
adapter = textadapter.text_adapter(data, field_names=False)
self.assert_equality(adapter[0].item(), (1, 2, 3))
data = StringIO('1\t2\t3\n')
adapter = textadapter.text_adapter(data, field_names=False)
self.assert_equality(adapter[0].item(), (1, 2, 3))
data = StringIO('1x2x3\n')
adapter = textadapter.text_adapter(data, field_names=False)
self.assert_equality(adapter[0].item(), (1, 2, 3))
# Test no delimiter in single field csv data
data = StringIO('aaa\nbbb\nccc')
array = textadapter.text_adapter(data,
field_names=False,
delimiter=None)[:]
assert_array_equal(
array,
np.array([('aaa', ), ('bbb', ), ('ccc', )], dtype=[('f0', 'O')]))
def test_no_whitespace_stripping(self):
data = StringIO('1 ,2 ,3 \n')
adapter = textadapter.text_adapter(data, field_names=False)
adapter.set_field_types({0: 'S3', 1: 'S3', 2: 'S3'})
assert_array_equal(adapter[:],
np.array([('1 ', '2 ', '3 ')], dtype='S3,S3,S3'))
data = StringIO(' 1, 2, 3\n')
adapter = textadapter.text_adapter(data, field_names=False)
adapter.set_field_types({0: 'S3', 1: 'S3', 2: 'S3'})
assert_array_equal(adapter[:],
np.array([(' 1', ' 2', ' 3')], dtype='S3,S3,S3'))
data = StringIO(' 1 , 2 , 3 \n')
adapter = textadapter.text_adapter(data, field_names=False)
adapter.set_field_types({0: 'S5', 1: 'S5', 2: 'S5'})
assert_array_equal(
adapter[:],
np.array([(' 1 ', ' 2 ', ' 3 ')], dtype='S5,S5,S5'))
data = StringIO('\t1\t,\t2\t,\t3\t\n')
adapter = textadapter.text_adapter(data, field_names=False)
adapter.set_field_types({0: 'S3', 1: 'S3', 2: 'S3'})
assert_array_equal(
adapter[:],
np.array([('\t1\t', '\t2\t', '\t3\t')], dtype='S3,S3,S3'))
def test_utf8_parsing(self):
# test single byte character
data = io.BytesIO(u'1,2,\u0033'.encode('utf_8'))
adapter = textadapter.text_adapter(data, field_names=False)
expected = np.array([('1', '2', '3')], dtype='u8,u8,u8')
assert_array_equal(adapter[:], expected)
# test multibyte character
data = io.BytesIO(u'1,2,\u2092'.encode('utf_8'))
adapter = textadapter.text_adapter(data, field_names=False)
expected = np.array([('1', '2', u'\u2092')], dtype='u8,u8,O')
assert_array_equal(adapter[:], expected)
def test_comments(self):
data = StringIO('1,2,3\n#4,5,6')
adapter = textadapter.text_adapter(data, field_names=False)
array = adapter[:]
self.assert_equality(array.size, 1)
self.assert_equality(array[0].item(), (1, 2, 3))
data = StringIO('1,2,3\n#4,5,6')
adapter = textadapter.text_adapter(data,
field_names=False,
comment=None)
array = adapter[:]
self.assert_equality(array.size, 2)
self.assert_equality(array[0].item(), ('1', 2, 3))
self.assert_equality(array[1].item(), ('#4', 5, 6))
def test_quoted_whitespace(self):
data = StringIO('"1 ","2 ","3 "\n')
adapter = textadapter.text_adapter(data, field_names=False)
adapter.set_field_types({0: 'S3', 1: 'S3', 2: 'S3'})
assert_array_equal(adapter[:],
np.array([('1 ', '2 ', '3 ')], dtype='S3,S3,S3'))
data = StringIO('"\t1\t"\t"\t2\t"\t"\t3\t"\n')
adapter = textadapter.text_adapter(data,
field_names=False,
delimiter='\t')
adapter.set_field_types({0: 'S3', 1: 'S3', 2: 'S3'})
assert_array_equal(
adapter[:],
np.array([('\t1\t', '\t2\t', '\t3\t')], dtype='S3,S3,S3'))
def test_generators(self):
def int_generator(num_recs):
for i in range(num_recs):
yield ','.join([
str(i * 5),
str(i * 5 + 1),
str(i * 5 + 2),
str(i * 5 + 3),
str(i * 5 + 4)
])
adapter = textadapter.text_adapter(int_generator(self.num_records),
field_names=False)
array = adapter[:]
self.assert_equality(array.size, self.num_records)
record = [x for x in range(0, 5)]
for i in range(0, self.num_records):
self.assert_equality(array[i].item(), tuple(record))
record[0] += 5
record[1] += 5
record[2] += 5
record[3] += 5
record[4] += 5
def test_converters(self):
data = StringIO()
generate_dataset(data, IntIter(), ',', self.num_records)
adapter = textadapter.text_adapter(data,
delimiter=',',
field_names=False)
#adapter.set_field_types({0:'u4', 1:'u4', 2:'u4', 3:'u4', 4:'u4'})
def increment(input_str):
return int(input_str) + 1
def double(input_str):
return int(input_str) + int(input_str)
if sys.platform == 'win32' and tuple.__itemsize__ == 8:
# TODO: there problems below here 64-bit Windows, I get
# OverflowError: can't convert negative value to unigned PY_LONG_LONG
return
adapter.set_converter(0, increment)
adapter.set_converter('f1', double)
array = adapter[:]
self.assert_equality(array.size, self.num_records)
record = [1, 2, 2, 3, 4]
for i in range(0, self.num_records):
self.assert_equality(array[i].item(), tuple(record))
record[0] += 5
record[1] = (10 * (i + 1)) + 2
record[2] += 5
record[3] += 5
record[4] += 5
def test_num_records(self):
data = StringIO(
'0,1\n2,3\n4,5\n6,7\n8,9\n10,11\n12,13\n14,15\n16,17\n18,19')
adapter = textadapter.text_adapter(data,
field_names=False,
num_records=2)
assert_array_equal(adapter[:], np.array([(0, 1), (2, 3)],
dtype='u8,u8'))
def test_string_parsing(self):
data = StringIO('1,2,3\n')
adapter = textadapter.text_adapter(data, field_names=False)
adapter.set_field_types({0: 'S5', 1: 'S5', 2: 'S5'})
assert_array_equal(adapter[:],
np.array([('1', '2', '3')], dtype='S5,S5,S5'))
data = io.StringIO(u'1,2,3\n')
adapter = textadapter.text_adapter(data, field_names=False)
adapter.set_field_types({0: 'S5', 1: 'S5', 2: 'S5'})
assert_array_equal(adapter[:],
np.array([('1', '2', '3')], dtype='S5,S5,S5'))
data = io.BytesIO(b'1,2,3\n')
adapter = textadapter.text_adapter(data, field_names=False)
adapter.set_field_types({0: 'S5', 1: 'S5', 2: 'S5'})
assert_array_equal(adapter[:],
np.array([('1', '2', '3')], dtype='S5,S5,S5'))
def test_spaces_around_numeric_values(self):
data = StringIO(' 1 , -2 , 3.3 , -4.4 \n 5 , -6 , 7.7 , -8.8 ')
adapter = textadapter.text_adapter(data, field_names=False)
adapter.set_field_types({0: 'u4', 1: 'i8', 2: 'f4', 3: 'f8'})
array = adapter[:]
control = np.array([(1, -2, 3.3, -4.4), (5, -6, 7.7, -8.8)],
dtype='u4,i8,f4,f8')
assert_array_equal(array, control)
def test_csv(self):
# Test skipping blank lines
data = StringIO('1,2,3\n\n4,5,6')
adapter = textadapter.text_adapter(data, field_names=False)
array = adapter[:]
assert_array_equal(
array,
np.array([(1, 2, 3), (4, 5, 6)],
dtype=[('f0', '<u8'), ('f1', '<u8'), ('f2', '<u8')]))
def test_gzip_index(self):
num_records = 1000000
data = StringIO()
generate_dataset(data, IntIter(), ',', num_records)
#if sys.version > '3':
if True:
dataz = io.BytesIO()
else:
dataz = StringIO()
gzip_output = gzip.GzipFile(fileobj=dataz, mode='wb')
#if sys.version > '3':
if True:
gzip_output.write(data.getvalue().encode('utf8'))
else:
gzip_output.write(data.getvalue())
gzip_output.close()
dataz.seek(0)
# test explicit index building
adapter = textadapter.text_adapter(dataz,
compression='gzip',
delimiter=',',
field_names=False,
infer_types=False)
adapter.set_field_types({0: 'u4', 1: 'u4', 2: 'u4', 3: 'u4', 4: 'u4'})
adapter.create_index()
self.assert_equality(adapter[0].item(),
tuple([(0 * 5) + x for x in range(5)]))
self.assert_equality(adapter[10].item(),
tuple([(10 * 5) + x for x in range(5)]))
self.assert_equality(adapter[100].item(),
tuple([(100 * 5) + x for x in range(5)]))
self.assert_equality(adapter[1000].item(),
tuple([(1000 * 5) + x for x in range(5)]))
self.assert_equality(adapter[10000].item(),
tuple([(10000 * 5) + x for x in range(5)]))
self.assert_equality(adapter[100000].item(),
tuple([(100000 * 5) + x for x in range(5)]))
self.assert_equality(
adapter[num_records - 1].item(),
tuple([((num_records - 1) * 5) + x for x in range(5)]))
#self.assert_equality(adapter[-1].item(), tuple(expected_values))
# test 'trouble' records that have caused crashes in the past
self.assert_equality(adapter[290000].item(),
tuple([(290000 * 5) + x for x in range(5)]))
self.assert_equality(adapter[818000].item(),
tuple([(818000 * 5) + x for x in range(5)]))
# test implicitly creating disk index on the fly
# JNB: not implemented yet
'''adapter = textadapter.text_adapter(dataz, compression='gzip', delimiter=',', field_names=False, infer_types=False, indexing=True, index_filename='test.idx')
def test_adapter_factory(self):
data = StringIO("1,2,3")
adapter = textadapter.text_adapter(data,
"csv",
delimiter=',',
field_names=False,
infer_types=False)
self.assertTrue(isinstance(adapter, textadapter.CSVTextAdapter))
self.assertRaises(textadapter.AdapterException,
textadapter.text_adapter, data, "foobar")
def test_stepping(self):
data = StringIO(
'0,1\n2,3\n4,5\n6,7\n8,9\n10,11\n12,13\n14,15\n16,17\n18,19')
adapter = textadapter.text_adapter(data, field_names=False)
assert_array_equal(
adapter[::2],
np.array([(0, 1), (4, 5), (8, 9), (12, 13), (16, 17)],
dtype='u8,u8'))
assert_array_equal(
adapter[::3],
np.array([(0, 1), (6, 7), (12, 13), (18, 19)], dtype='u8,u8'))
def test_64bit_ints(self):
data = StringIO(
str((2**63) - 1) + ',' + str(((2**63) - 1) * -1) + ',' +
str((2**64) - 1))
adapter = textadapter.text_adapter(data,
delimiter=',',
field_names=False,
infer_types=False)
adapter.set_field_types({0: 'i8', 1: 'i8', 2: 'u8'})
array = adapter.to_array()
self.assert_equality(array[0].item(),
((2**63) - 1, ((2**63) - 1) * -1, (2**64) - 1))
def test_float_conversion(self):
data = StringIO('10,1.333,-1.23,10.0E+2,999.9e-2')
adapter = textadapter.text_adapter(data,
field_names=False,
infer_types=False)
adapter.set_field_types(dict(zip(range(5), ['f8'] * 5)))
array = adapter[0]
#self.assert_equality(array[0].item(), (10.0,1.333,-1.23,1000.0,9.999))
self.assertAlmostEqual(array[0][0], 10.0)
self.assertAlmostEqual(array[0][1], 1.333)
self.assertAlmostEqual(array[0][2], -1.23)
self.assertAlmostEqual(array[0][3], 1000.0)
self.assertAlmostEqual(array[0][4], 9.999)
def test_escapechar(self):
data = StringIO('1,2\\2,3\n4,5\\5\\5,6')
array = textadapter.text_adapter(data, field_names=False)[:]
assert_array_equal(
array, np.array([(1, 22, 3), (4, 555, 6)], dtype='u8,u8,u8'))
data = StringIO('\\1,2,3\n4,5,6\\')
array = textadapter.text_adapter(data, field_names=False)[:]
assert_array_equal(array,
np.array([(1, 2, 3), (4, 5, 6)], dtype='u8,u8,u8'))
data = StringIO('a,b\\,b,c\na,b\\,b\\,b,c')
array = textadapter.text_adapter(data, field_names=False)[:]
assert_array_equal(
array,
np.array([('a', 'b,b', 'c'), ('a', 'b,b,b', 'c')], dtype='O,O,O'))
data = StringIO('a,bx,b,c\na,bx,bx,b,c')
array = textadapter.text_adapter(data, field_names=False,
escape='x')[:]
assert_array_equal(
array,
np.array([('a', 'b,b', 'c'), ('a', 'b,b,b', 'c')], dtype='O,O,O'))
def test_header_footer(self):
data = StringIO('0,1,2,3,4\n5,6,7,8,9\n10,11,12,13,14')
adapter = textadapter.text_adapter(data, header=1, field_names=False)
adapter.field_types = dict(zip(range(5), ['u4'] * 5))
assert_array_equal(
adapter[:],
np.array([(5, 6, 7, 8, 9), (10, 11, 12, 13, 14)],
dtype='u4,u4,u4,u4,u4'))
data.seek(0)
adapter = textadapter.text_adapter(data, header=2, field_names=False)
adapter.field_types = dict(zip(range(5), ['u4'] * 5))
assert_array_equal(
adapter[:], np.array([(10, 11, 12, 13, 14)],
dtype='u4,u4,u4,u4,u4'))
data.seek(0)
adapter = textadapter.text_adapter(data, header=1, field_names=True)
adapter.field_types = dict(zip(range(5), ['u4'] * 5))
assert_array_equal(
adapter[:],
np.array([(10, 11, 12, 13, 14)],
dtype=[('5', 'u4'), ('6', 'u4'), ('7', 'u4'), ('8', 'u4'),
('9', 'u4')]))
def test_slicing(self):
data = StringIO()
generate_dataset(data, IntIter(), ',', self.num_records)
adapter = textadapter.text_adapter(data, field_names=False)
adapter.set_field_types({0: 'u4', 1: 'u4', 2: 'u4', 3: 'u4', 4: 'u4'})
assert_array_equal(adapter[0],
np.array([(0, 1, 2, 3, 4)], dtype='u4,u4,u4,u4,u4'))
expected_values = [((self.num_records - 1) * 5) + x for x in range(5)]
self.assert_equality(adapter[self.num_records - 1].item(),
tuple(expected_values))
#adapter.create_index()
#self.assert_equality(adapter[-1].item(), tuple(expected_values))
self.assert_equality(adapter['f0'][0].item(), (0, ))
self.assert_equality(adapter['f4'][1].item(), (9, ))
#self.assert_equality(adapter[self.num_records-1]['f4'], (self.num_records*5)-1)
array = adapter[:]
record = [x for x in range(0, 5)]
self.assert_equality(array.size, self.num_records)
for i in range(0, self.num_records):
self.assert_equality(array[i].item(), tuple(record))
record = [x + 5 for x in record]
array = adapter[:-1]
record = [x for x in range(0, 5)]
self.assert_equality(array.size, self.num_records - 1)
for i in range(0, self.num_records - 1):
self.assert_equality(array[i].item(), tuple(record))
record = [x + 5 for x in record]
array = adapter[0:10]
self.assert_equality(array.size, 10)
record = [x for x in range(0, 5)]
for i in range(0, 10):
self.assert_equality(array[i].item(), tuple(record))
record = [x + 5 for x in record]
array = adapter[1:]
self.assert_equality(array.size, self.num_records - 1)
record = [x for x in range(5, 10)]
for i in range(0, self.num_records - 1):
self.assert_equality(array[i].item(), tuple(record))
record = [x + 5 for x in record]
array = adapter[0:10:2]
self.assert_equality(array.size, 5)
record = [x for x in range(0, 5)]
for i in range(0, 5):
self.assert_equality(array[i].item(), tuple(record))
record = [x + 10 for x in record]
array = adapter[['f0', 'f4']][:]
record = [0, 4]
self.assert_equality(array.size, self.num_records)
for i in range(0, self.num_records):
self.assert_equality(array[i].item(), tuple(record))
record = [x + 5 for x in record]
adapter.field_filter = [0, 'f4']
array = adapter[:]
record = [0, 4]
self.assert_equality(array.size, self.num_records)
for i in range(0, self.num_records):
self.assert_equality(array[i].item(), tuple(record))
record = [x + 5 for x in record]
adapter.field_filter = None
array = adapter[:]
record = [0, 1, 2, 3, 4]
self.assert_equality(array.size, self.num_records)
for i in range(0, self.num_records):
self.assert_equality(array[i].item(), tuple(record))
record = [x + 5 for x in record]
try:
adapter[self.num_records]
except textadapter.AdapterIndexError:
pass
else:
self.fail('AdaperIndexError not thrown')
try:
adapter[0:self.num_records + 1]
except textadapter.AdapterIndexError:
pass
else:
self.fail('AdaperIndexError not thrown')
def test_missing_fill_values(self):
data = StringIO()
generate_dataset(data, MissingValuesIter(), ',', self.num_records)
adapter = textadapter.text_adapter(data,
delimiter=',',
field_names=False,
infer_types=False)
adapter.set_field_types({
'f0': 'u4',
1: 'u4',
2: 'u4',
3: 'u4',
'f4': 'u4'
})
adapter.set_missing_values({0: ['NA', 'NaN'], 'f4': ['xx', 'inf']})
adapter.set_fill_values({0: 99, 4: 999})
array = adapter[:]
self.assert_equality(array.size, self.num_records)
record = [x for x in range(0, 5)]
for i in range(0, self.num_records):
if i % 4 == 0 or i % 4 == 1:
record[0] = 99
record[4] = 999
else:
record[0] = record[1] - 1
record[4] = record[3] + 1
self.assert_equality(array[i].item(), tuple(record))
record[1] += 5
record[2] += 5
record[3] += 5
data.seek(0)
adapter = textadapter.text_adapter(data,
delimiter=',',
field_names=False,
infer_types=True)
adapter.set_missing_values({0: ['NA', 'NaN'], 4: ['xx', 'inf']})
array = adapter[:]
self.assert_equality(array.size, self.num_records)
record = [x for x in range(0, 5)]
for i in range(0, self.num_records):
if i % 4 == 0 or i % 4 == 1:
record[0] = 0
record[4] = 0
else:
record[0] = record[1] - 1
record[4] = record[3] + 1
self.assert_equality(array[i].item(), tuple(record))
record[1] += 5
record[2] += 5
record[3] += 5
# Test missing field
data = StringIO('1,2,3\n4,5\n7,8,9')
adapter = textadapter.text_adapter(data, field_names=False)
adapter.field_types = {0: 'O', 1: 'O', 2: 'O'}
adapter.set_fill_values({0: np.nan, 1: np.nan, 2: np.nan})
array = adapter[:]
# NumPy assert_array_equal no longer supports mixed O/nan types
expected = [('1', '2', '3'), ('4', '5', np.nan), ('7', '8', '9')]
self.assert_equality(array.tolist(), expected)
def test_fixed_width(self):
data = StringIO()
generate_dataset(data, FixedWidthIter(), '', self.num_records)
adapter = textadapter.FixedWidthTextAdapter(data, [2, 3, 4, 5, 6],
field_names=False,
infer_types=False)
adapter.set_field_types({0: 'u4', 1: 'u4', 2: 'u4', 3: 'u4', 4: 'u4'})
array = adapter[:]
self.assert_equality(array.size, self.num_records)
record = [0, 0, 0, 0, 0]
for i in range(0, self.num_records):
self.assert_equality(array[i].item(), tuple(record))
record = [x + 1 for x in record]
if record[0] == 100:
record[0] = 0
if record[1] == 1000:
record[1] = 0
if record[2] == 10000:
record[2] = 0
if record[3] == 100000:
record[3] = 0
if record[4] == 1000000:
record[4] = 0
# Test skipping blank lines
data = StringIO(' 1 2 3\n\n 4 5 6')
adapter = textadapter.text_adapter(data,
parser='fixed_width',
field_widths=[2, 2, 2],
field_names=False)
array = adapter[:]
assert_array_equal(
array,
np.array([(1, 2, 3), (4, 5, 6)],
dtype=[('f0', '<u8'), ('f1', '<u8'), ('f2', '<u8')]))
# Test comment lines
data = StringIO('# 1 2 3\n 1 2 3\n# foo\n 4 5 6')
adapter = textadapter.text_adapter(data,
parser='fixed_width',
field_widths=[2, 2, 2],
field_names=False)
array = adapter[:]
assert_array_equal(
array,
np.array([(1, 2, 3), (4, 5, 6)],
dtype=[('f0', '<u8'), ('f1', '<u8'), ('f2', '<u8')]))
# Test field names line
data = StringIO(' a b c\n 1 2 3')
adapter = textadapter.text_adapter(data,
parser='fixed_width',
field_widths=[2, 2, 2],
field_names=True)
array = adapter[:]
assert_array_equal(
array,
np.array([(1, 2, 3)],
dtype=[('a', '<u8'), ('b', '<u8'), ('c', '<u8')]))
# Test field names line as comment line
data = StringIO('# a b c\n 1 2 3')
adapter = textadapter.text_adapter(data,
parser='fixed_width',
field_widths=[2, 2, 2],
field_names=True)
array = adapter[:]
assert_array_equal(
array,
np.array([(1, 2, 3)],
dtype=[('a', '<u8'), ('b', '<u8'), ('c', '<u8')]))
# Test incomplete field names line
data = StringIO(' a\n 1 2 3')
adapter = textadapter.text_adapter(data,
parser='fixed_width',
field_widths=[2, 2, 2],
field_names=True)
array = adapter[:]
assert_array_equal(
array,
np.array([(1, 2, 3)],
dtype=[('a', '<u8'), ('f1', '<u8'), ('f2', '<u8')]))
def test_regex(self):
data = StringIO()
generate_dataset(data, IntIter(), ',', self.num_records)
adapter = textadapter.RegexTextAdapter(
data,
'([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*)\n',
field_names=False,
infer_types=False)
adapter.set_field_types({0: 'u4', 1: 'u4', 2: 'u4', 3: 'u4', 4: 'u4'})
array = adapter[:]
self.assert_equality(array.size, self.num_records)
record = [x for x in range(0, 5)]
for i in range(0, self.num_records):
self.assert_equality(array[i].item(), tuple(record))
record = [x + 5 for x in record]
# Test skipping blank lines
data = StringIO('1 2 3\n\n4 5 6')
adapter = textadapter.text_adapter(
data,
parser='regex',
regex_string='([0-9]) ([0-9]) ([0-9])',
field_names=False)
array = adapter[:]
assert_array_equal(
array,
np.array([(1, 2, 3), (4, 5, 6)],
dtype=[('f0', '<u8'), ('f1', '<u8'), ('f2', '<u8')]))
# Test comment lines
data = StringIO('#1 2 3\n1 2 3\n# foo\n4 5 6')
adapter = textadapter.text_adapter(
data,
parser='regex',
regex_string='([0-9]) ([0-9]) ([0-9])',
field_names=False)
array = adapter[:]
assert_array_equal(
array,
np.array([(1, 2, 3), (4, 5, 6)],
dtype=[('f0', '<u8'), ('f1', '<u8'), ('f2', '<u8')]))
# Test field names line
data = StringIO('a b c\n4 5 6')
adapter = textadapter.text_adapter(
data,
parser='regex',
regex_string='([0-9]) ([0-9]) ([0-9])',
field_names=True)
array = adapter[:]
assert_array_equal(
array,
np.array([(4, 5, 6)],
dtype=[('a', '<u8'), ('b', '<u8'), ('c', '<u8')]))
# Test field names line as comment line
data = StringIO('#a b c\n4 5 6')
adapter = textadapter.text_adapter(
data,
parser='regex',
regex_string='([0-9]) ([0-9]) ([0-9])',
field_names=True)
array = adapter[:]
assert_array_equal(
array,
np.array([(4, 5, 6)],
dtype=[('a', '<u8'), ('b', '<u8'), ('c', '<u8')]))
# Test incomplete field names line
data = StringIO('a b\n4 5 6')
adapter = textadapter.text_adapter(
data,
parser='regex',
regex_string='([0-9]) ([0-9]) ([0-9])',
field_names=True)
array = adapter[:]
assert_array_equal(
array,
np.array([(4, 5, 6)],
dtype=[('a', '<u8'), ('b', '<u8'), ('f2', '<u8')]))
# Test field names line that doesn't match regex
data = StringIO('a b c\n1 2 3 4 5 6')
adapter = textadapter.text_adapter(
data,
parser='regex',
regex_string='([0-9\s]+) ([0-9\s]+) ([0-9\s]+)',
field_names=True)
array = adapter[:]
assert_array_equal(
array,
np.array([('1 2', '3 4', '5 6')],
dtype=[('a', 'O'), ('b', 'O'), ('c', 'O')]))
def test_index(self):
if sys.platform == 'win32':
# TODO: this test fails on Windows because of file lock problems
return
num_records = 100000
expected_values = [((num_records - 1) * 5) + x for x in range(5)]
data = StringIO()
generate_dataset(data, IntIter(), ',', num_records)
# test explicit index building
adapter = textadapter.text_adapter(data,
delimiter=',',
field_names=False,
infer_types=False)
adapter.set_field_types({0: 'u4', 1: 'u4', 2: 'u4', 3: 'u4', 4: 'u4'})
adapter.create_index()
self.assert_equality(adapter[0].item(),
tuple([(0 * 5) + x for x in range(5)]))
self.assert_equality(adapter[10].item(),
tuple([(10 * 5) + x for x in range(5)]))
self.assert_equality(adapter[100].item(),
tuple([(100 * 5) + x for x in range(5)]))
self.assert_equality(adapter[1000].item(),
tuple([(1000 * 5) + x for x in range(5)]))
self.assert_equality(adapter[10000].item(),
tuple([(10000 * 5) + x for x in range(5)]))
self.assert_equality(
adapter[num_records - 1].item(),
tuple([((num_records - 1) * 5) + x for x in range(5)]))
#self.assert_equality(adapter[-1].item(), tuple(expected_values))
# test implicitly creating disk index on the fly
if os.path.exists('test.idx'):
os.remove('test.idx')
data.seek(0)
adapter = textadapter.text_adapter(data,
delimiter=',',
field_names=False,
infer_types=False,
index_name='test.idx')
adapter.set_field_types({0: 'u4', 1: 'u4', 2: 'u4', 3: 'u4', 4: 'u4'})
adapter.to_array()
self.assert_equality(adapter[0].item(),
tuple([(0 * 5) + x for x in range(5)]))
self.assert_equality(adapter[10].item(),
tuple([(10 * 5) + x for x in range(5)]))
self.assert_equality(adapter[100].item(),
tuple([(100 * 5) + x for x in range(5)]))
self.assert_equality(adapter[1000].item(),
tuple([(1000 * 5) + x for x in range(5)]))
self.assert_equality(adapter[10000].item(),
tuple([(10000 * 5) + x for x in range(5)]))
self.assert_equality(
adapter[num_records - 1].item(),
tuple([((num_records - 1) * 5) + x for x in range(5)]))
#self.assert_equality(adapter[-1].item(), tuple(expected_values))
adapter.close()
# test loading disk index
data.seek(0)
adapter2 = textadapter.text_adapter(data,
delimiter=',',
field_names=False,
infer_types=False,
index_name='test.idx')
adapter2.set_field_types({0: 'u4', 1: 'u4', 2: 'u4', 3: 'u4', 4: 'u4'})
self.assert_equality(adapter2[0].item(),
tuple([(0 * 5) + x for x in range(5)]))
self.assert_equality(adapter2[10].item(),
tuple([(10 * 5) + x for x in range(5)]))
self.assert_equality(adapter2[100].item(),
tuple([(100 * 5) + x for x in range(5)]))
self.assert_equality(adapter2[1000].item(),
tuple([(1000 * 5) + x for x in range(5)]))
self.assert_equality(adapter2[10000].item(),
tuple([(10000 * 5) + x for x in range(5)]))
self.assert_equality(
adapter2[num_records - 1].item(),
tuple([((num_records - 1) * 5) + x for x in range(5)]))
#self.assert_equality(adapter2[-1].item(), tuple(expected_values))
adapter.close()
os.remove('test.idx')
def test_json(self):
# Test json number
data = StringIO('{"id":123}')
adapter = textadapter.text_adapter(data, parser='json')
array = adapter[:]
assert_array_equal(array, np.array([(123, )], dtype=[('id', 'u8')]))
# Test json number
data = StringIO('{"id":"xxx"}')
adapter = textadapter.text_adapter(data, parser='json')
array = adapter[:]
assert_array_equal(array, np.array([('xxx', )], dtype=[('id', 'O')]))
# Test multiple values
data = StringIO('{"id":123, "name":"xxx"}')
adapter = textadapter.text_adapter(data, parser='json')
array = adapter[:]
assert_array_equal(
array,
np.array([(
123,
'xxx',
)], dtype=[('id', 'u8'), ('name', 'O')]))
# Test multiple records
data = StringIO('[{"id":123, "name":"xxx"}, {"id":456, "name":"yyy"}]')
adapter = textadapter.text_adapter(data, parser='json')
array = adapter[:]
assert_array_equal(
array,
np.array([(
123,
'xxx',
), (456, 'yyy')],
dtype=[('id', 'u8'), ('name', 'O')]))
# Test multiple objects separated by newlines
data = StringIO('{"id":123, "name":"xxx"}\n{"id":456, "name":"yyy"}')
adapter = textadapter.text_adapter(data, parser='json')
array = adapter[:]
assert_array_equal(
array,
np.array([(
123,
'xxx',
), (456, 'yyy')],
dtype=[('id', 'u8'), ('name', 'O')]))
data = StringIO('{"id":123, "name":"xxx"}\n')
adapter = textadapter.text_adapter(data, parser='json')
array = adapter[:]
assert_array_equal(
array,
np.array([(
123,
'xxx',
)], dtype=[('id', 'u8'), ('name', 'O')]))
# JNB: broken; should be really be supporting the following json inputs?
'''
# Test subarrays
data = StringIO('{"id":123, "names":["xxx","yyy","zzz"]}')
adapter = textadapter.text_adapter(data, parser='json')
array = adapter[:]
assert_array_equal(array, np.array([(123, 'xxx', 'yyy', 'zzz',)],
dtype=[('f0', 'u8'), ('f1', 'O'), ('f2', 'O'), ('f3', 'O')]))
# Test subobjects
data = StringIO('{"id":123, "names":{"a":"xxx", "b":"yyy", "c":"zzz"}}')
adapter = textadapter.text_adapter(data, parser='json')
array = adapter[:]
assert_array_equal(array, np.array([(123, 'xxx', 'yyy', 'zzz',)],
dtype=[('f0', 'u8'), ('f1', 'O'), ('f2', 'O'), ('f3', 'O')]))
'''
# Test ranges
data = StringIO('{"id": 1, "name": "www"}\n'
'{"id": 2, "name": "xxx"}\n'
'{"id": 3, "name": "yyy"}\n'
'{"id": 4, "name": "zzz"}')
adapter = textadapter.text_adapter(data, parser='json')
array = adapter[2:4]
assert_array_equal(
array,
np.array([(3, 'yyy'), (4, 'zzz')],
dtype=[('id', 'u8'), ('name', 'O')]))
# Test column order
data = StringIO('{"xxx": 1, "aaa": 2}\n')
adapter = textadapter.text_adapter(data, parser='json')
array = adapter[:]
assert_array_equal(
array, np.array([(1, 2)], dtype=[('xxx', 'u8'), ('aaa', 'u8')]))
# Test field filter
data = StringIO('{"id": 1, "name": "www"}\n'
'{"id": 2, "name": "xxx"}\n'
'{"id": 3, "name": "yyy"}\n'
'{"id": 4, "name": "zzz"}')
adapter = textadapter.text_adapter(data, parser='json')
adapter.field_filter = ['name']
array = adapter[:]
assert_array_equal(
array,
np.array([('www', ), ('xxx', ), ('yyy', ), ('zzz', )],
dtype=[('name', 'O')]))
def genfromtxt(fname,
dtype=float,
comments='#',
delimiter=None,
skiprows=0,
skip_header=0,
skip_footer=0,
converters=None,
missing='',
missing_values=None,
filling_values=None,
usecols=None,
names=None,
excludelist=None,
deletechars=None,
replace_space='_',
autostrip=False,
case_sensitive=True,
defaultfmt="f%i",
unpack=None,
usemask=False,
loose=True,
invalid_raise=True):
"""
Load data from a text file, with missing values handled as specified.
Each line past the first `skip_header` lines is split at the `delimiter`
character, and characters following the `comments` character are discarded.
Parameters
----------
fname : file or str
File, filename, or generator to read. If the filename extension is
`.gz` or `.bz2`, the file is first decompressed. Note that
generators must return byte strings in Python 3k.
dtype : dtype, optional
Data type of the resulting array.
If None, the dtypes will be determined by the contents of each
column, individually.
comments : str, optional
The character used to indicate the start of a comment.
All the characters occurring on a line after a comment are discarded
delimiter : str, int, or sequence, optional
The string used to separate values. By default, any consecutive
whitespaces act as delimiter. An integer or sequence of integers
can also be provided as width(s) of each field.
skip_header : int, optional
The numbers of lines to skip at the beginning of the file.
skip_footer : int, optional
The numbers of lines to skip at the end of the file
converters : variable, optional
The set of functions that convert the data of a column to a value.
The converters can also be used to provide a default value
for missing data: ``converters = {3: lambda s: float(s or 0)}``.
missing_values : variable, optional
The set of strings corresponding to missing data.
filling_values : variable, optional
The set of values to be used as default when the data are missing.
usecols : sequence, optional
Which columns to read, with 0 being the first. For example,
``usecols = (1, 4, 5)`` will extract the 2nd, 5th and 6th columns.
names : {None, True, str, sequence}, optional
If `names` is True, the field names are read from the first valid line
after the first `skip_header` lines.
If `names` is a sequence or a single-string of comma-separated names,
the names will be used to define the field names in a structured dtype.
If `names` is None, the names of the dtype fields will be used, if any.
excludelist : sequence, optional
A list of names to exclude. This list is appended to the default list
['return','file','print']. Excluded names are appended an underscore:
for example, `file` would become `file_`.
deletechars : str, optional
A string combining invalid characters that must be deleted from the
names.
defaultfmt : str, optional
A format used to define default field names, such as "f%i" or "f_%02i".
autostrip : bool, optional
Whether to automatically strip white spaces from the variables.
replace_space : char, optional
Character(s) used in replacement of white spaces in the variables
names. By default, use a '_'.
case_sensitive : {True, False, 'upper', 'lower'}, optional
If True, field names are case sensitive.
If False or 'upper', field names are converted to upper case.
If 'lower', field names are converted to lower case.
unpack : bool, optional
If True, the returned array is transposed, so that arguments may be
unpacked using ``x, y, z = loadtxt(...)``
usemask : bool, optional
If True, return a masked array.
If False, return a regular array.
invalid_raise : bool, optional
If True, an exception is raised if an inconsistency is detected in the
number of columns.
If False, a warning is emitted and the offending lines are skipped.
Returns
-------
out : ndarray
Data read from the text file. If `usemask` is True, this is a
masked array.
See Also
--------
numpy.loadtxt : equivalent function when no data is missing.
Notes
-----
* When spaces are used as delimiters, or when no delimiter has been given
as input, there should not be any missing data between two fields.
* When the variables are named (either by a flexible dtype or with `names`,
there must not be any header in the file (else a ValueError
exception is raised).
* Individual values are not stripped of spaces by default.
When using a custom converter, make sure the function does remove spaces.
References
----------
.. [1] Numpy User Guide, section `I/O with Numpy
<http://docs.scipy.org/doc/numpy/user/basics.io.genfromtxt.html>`_.
Examples
---------
>>> from StringIO import StringIO
>>> import numpy as np
Comma delimited file with mixed dtype
>>> s = StringIO("1,1.3,abcde")
>>> data = np.genfromtxt(s, dtype=[('myint','i8'),('myfloat','f8'),
... ('mystring','S5')], delimiter=",")
>>> data
array((1, 1.3, 'abcde'),
dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', '|S5')])
Using dtype = None
>>> s.seek(0) # needed for StringIO example only
>>> data = np.genfromtxt(s, dtype=None,
... names = ['myint','myfloat','mystring'], delimiter=",")
>>> data
array((1, 1.3, 'abcde'),
dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', '|S5')])
Specifying dtype and names
>>> s.seek(0)
>>> data = np.genfromtxt(s, dtype="i8,f8,S5",
... names=['myint','myfloat','mystring'], delimiter=",")
>>> data
array((1, 1.3, 'abcde'),
dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', '|S5')])
An example with fixed-width columns
>>> s = StringIO("11.3abcde")
>>> data = np.genfromtxt(s, dtype=None, names=['intvar','fltvar','strvar'],
... delimiter=[1,3,5])
>>> data
array((1, 1.3, 'abcde'),
dtype=[('intvar', '<i8'), ('fltvar', '<f8'), ('strvar', '|S5')])
"""
# Py3 data conversions to bytes, for convenience
#comments = asbytes(comments)
#if isinstance(delimiter, unicode):
# delimiter = asbytes(delimiter)
#if isinstance(missing, unicode):
# missing = asbytes(missing)
#if isinstance(missing_values, (unicode, list, tuple)):
# missing_values = asbytes_nested(missing_values)
if usemask:
from numpy.ma import MaskedArray, make_mask_descr
# Check the input dictionary of converters
user_converters = converters or {}
if not isinstance(user_converters, dict):
errmsg = "The input argument 'converter' should be a valid dictionary "\
"(got '%s' instead)"
raise TypeError(errmsg % type(user_converters))
# Initialize the filehandle, the LineSplitter and the NameValidator
own_fhd = False
try:
if isinstance(fname, basestring):
fhd = iter(np.lib._datasource.open(fname, 'rbU'))
own_fhd = True
else:
fhd = iter(fname)
except TypeError:
raise TypeError("fname mustbe a string, filehandle, or generator. "\
"(got %s instead)" % type(fname))
validate_names = NameValidator(excludelist=excludelist,
deletechars=deletechars,
case_sensitive=case_sensitive,
replace_space=replace_space)
# Get the first valid lines after the first skiprows ones ..
if skiprows:
warnings.warn(\
"The use of `skiprows` is deprecated, it will be removed in numpy 2.0.\n" \
"Please use `skip_header` instead.",
DeprecationWarning)
skip_header = skiprows
set_names = False
if names is True:
set_names = True
infer_types = False
if dtype is None:
infer_types = True
whitespace_delims = False
if delimiter is None:
delimiter = ' '
whitespace_delims = True
compression = None
if isinstance(fname, basestring) and fname[-3:] == '.gz':
compression = 'gzip'
try:
if isinstance(delimiter, basestring):
adapter = textadapter.text_adapter(
fname,
parser='csv',
delimiter=delimiter,
comment=comments,
header=skip_header,
footer=skip_footer,
compression=compression,
field_names=set_names,
infer_types=True,
whitespace_delims=whitespace_delims)
elif isinstance(delimiter, int) or isinstance(delimiter,
(list, tuple)):
adapter = textadapter.text_adapter(fname,
parser='fixed_width',
field_widths=delimiter,
comment=comments,
header=skip_header,
footer=skip_footer,
field_names=set_names,
infer_types=True)
except EOFError:
return np.array([])
field_names = None
if isinstance(names, basestring):
field_names = [name.strip() for name in names.split(',')]
elif isinstance(names, tuple):
field_names = list(names)
elif isinstance(names, list):
field_names = names
elif set_names is True:
field_names = adapter.field_names
if usecols is None:
usecols = [x for x in range(0, adapter.field_count)]
elif isinstance(usecols, basestring) and field_names is None:
raise ValueError('usecols contains unknown field names')
elif isinstance(usecols, basestring):
if field_names is None:
raise ValueError('usecols contains unknown field names')
else:
usecols = [
field_names.index(name.strip()) for name in usecols.split(',')
]
elif isinstance(usecols, (list, tuple)):
if len(usecols) == 0:
raise ValueError('usecols must contain at least one col')
tempCols = []
for col in usecols:
if isinstance(col, basestring) and field_names is None:
raise ValueError('usecols contains unknown field names')
elif isinstance(col, basestring):
tempCols.append(field_names.index(col))
elif isinstance(col, int):
tempCols.append(col)
else:
raise ValueError(
'usecols must contain either field numbers or field names')
usecols = tempCols
elif isinstance(usecols, int_types):
usecols = [usecols]
elif isinstance(usecols, numpy.ndarray):
usecols = usecols.tolist()
else:
usecols = list(usecols)
# adjust negative indices
for i, col in enumerate(usecols):
if col < 0:
usecols[i] = adapter.field_count + col
if isinstance(field_names,
(list, dict)) and len(field_names) < len(usecols):
field_names.extend([''] * (len(usecols) - len(field_names)))
# Process the deprecated `missing`
if missing != asbytes(''):
warnings.warn(\
"The use of `missing` is deprecated, it will be removed in Numpy 2.0.\n" \
"Please use `missing_values` instead.",
DeprecationWarning)
missing_values = missing
# Initialize the output lists ...
# ... rows
rows = []
append_to_rows = rows.append
# ... masks
if usemask:
masks = []
append_to_masks = masks.append
# ... invalid
invalid = []
append_to_invalid = invalid.append
# create valid dtype object
if isinstance(dtype, (list, tuple)):
dtype = [dt if isinstance(dt, tuple) else ('', dt) for dt in dtype]
dtype = np.dtype(dtype)
# create list of dtypes to send to TextAdapter
if dtype.names is None:
# create list of homogenous scalar dtypes from single scalar dtype
numFields = len(usecols)
dtypes = [dtype] * numFields
else:
# create list of scalar dtypes from struct dtype
dtypes, dtype_field_names = unpack_dtype(dtype)
if field_names is None:
field_names = dtype_field_names
else:
s = set(field_names)
# if all entries in field_names are empty, use dtype field names
if len(s) == 1 and s.pop() == '':
field_names = dtype_field_names
# use field names from dtype if field names were not specified by user
# and not read from first line in file
#if names is None and dtype.names is not None:
# field_names = dtype.names
if field_names is not None:
adapter.field_names = field_names
if infer_types is False:
adapter.set_field_types(types=dict(zip(usecols, dtypes)))
if converters is not None:
for field, converter in converters.items():
adapter.set_converter(field, converter)
if isinstance(missing_values, basestring):
values = missing_values.split(',')
missing_values = {}
for i in range(adapter.field_count):
missing_values[i] = values
if missing_values is not None:
adapter.set_missing_values(missing_values)
if isinstance(filling_values, basestring):
filling_values = filling_values.split(',')
if filling_values is not None:
filling_values = dict([(key, value)
for key, value in filling_values.items()
if key in usecols])
adapter.set_fill_values(filling_values, loose)
try:
array = adapter[usecols][:]
except DataTypeError:
raise ValueError
if own_fhd:
fhd.close()
# Adapter returns an array with struct dtype.
# If no field names were specified or read from file,
# and specified dtype is scalar, reset array to scalar dtype.
if dtype.fields is None and field_names is None and set_names is False:
# Can't set final dtype to scalar if struct dtype includes objects
if array.dtype.fields is not None \
and np.object_ not in [x[0] for x in array.dtype.fields.values()]:
array.dtype = dtype
# If no fields were read, we want to keep this a 1-d empty array.
# Otherwise, set the proper shape.
if adapter.field_count > 0:
array.shape = (adapter.size, len(usecols))
#elif dtype.names is None and isinstance(names, list):
# array.dtype = zip([names[i] for i in usecols], [dtype]*len(usecols))
# Construct the final array
#if usemask:
# array = array.view(MaskedArray)
# array._mask = outputmask
if unpack:
return array.squeeze().T
return array.squeeze()
def loadtxt(fname,
dtype=float,
comments='#',
delimiter=None,
converters=None,
skiprows=0,
usecols=None,
unpack=False,
ndmin=0):
"""
Load data from a text file.
Each row in the text file must have the same number of values.
Parameters
----------
fname : file or str
File, filename, or generator to read. If the filename extension is
``.gz`` or ``.bz2``, the file is first decompressed. Note that
generators should return byte strings for Python 3k.
dtype : data-type, optional
Data-type of the resulting array; default: float. If this is a
record data-type, the resulting array will be 1-dimensional, and
each row will be interpreted as an element of the array. In this
case, the number of columns used must match the number of fields in
the data-type.
comments : str, optional
The character used to indicate the start of a comment;
default: '#'.
delimiter : str, optional
The string used to separate values. By default, this is any
whitespace.
converters : dict, optional
A dictionary mapping column number to a function that will convert
that column to a float. E.g., if column 0 is a date string:
``converters = {0: datestr2num}``. Converters can also be used to
provide a default value for missing data (but see also `genfromtxt`):
``converters = {3: lambda s: float(s.strip() or 0)}``. Default: None.
skiprows : int, optional
Skip the first `skiprows` lines; default: 0.
usecols : sequence, optional
Which columns to read, with 0 being the first. For example,
``usecols = (1,4,5)`` will extract the 2nd, 5th and 6th columns.
The default, None, results in all columns being read.
unpack : bool, optional
If True, the returned array is transposed, so that arguments may be
unpacked using ``x, y, z = loadtxt(...)``. When used with a record
data-type, arrays are returned for each field. Default is False.
ndmin : int, optional
The returned array will have at least `ndmin` dimensions.
Otherwise mono-dimensional axes will be squeezed.
Legal values: 0 (default), 1 or 2.
.. versionadded:: 1.6.0
Returns
-------
out : ndarray
Data read from the text file.
See Also
--------
load, fromstring, fromregex
genfromtxt : Load data with missing values handled as specified.
scipy.io.loadmat : reads MATLAB data files
Notes
-----
This function aims to be a fast reader for simply formatted files. The
`genfromtxt` function provides more sophisticated handling of, e.g.,
lines with missing values.
Examples
--------
>>> from StringIO import StringIO # StringIO behaves like a file object
>>> c = StringIO("0 1\\n2 3")
>>> np.loadtxt(c)
array([[ 0., 1.],
[ 2., 3.]])
>>> d = StringIO("M 21 72\\nF 35 58")
>>> np.loadtxt(d, dtype={'names': ('gender', 'age', 'weight'),
... 'formats': ('S1', 'i4', 'f4')})
array([('M', 21, 72.0), ('F', 35, 58.0)],
dtype=[('gender', '|S1'), ('age', '<i4'), ('weight', '<f4')])
>>> c = StringIO("1,0,2\\n3,0,4")
>>> x, y = np.loadtxt(c, delimiter=',', usecols=(0, 2), unpack=True)
>>> x
array([ 1., 3.])
>>> y
array([ 2., 4.])
"""
user_converters = converters
whitespace_delims = False
if delimiter is None:
whitespace_delims = True
compression = None
if isinstance(fname, basestring) and fname[-3:] == '.gz':
compression = 'gzip'
try:
adapter = textadapter.text_adapter(fname,
parser='csv',
delimiter=delimiter,
comment=comments,
header=skiprows,
compression=compression,
whitespace_delims=whitespace_delims,
field_names=False,
infer_types=False)
except EOFError:
array = numpy.array([], dtype=numpy.int64, ndmin=ndmin)
if ndmin == 2:
array = array.T
return array
if usecols is None:
usecols = [x for x in range(0, adapter.field_count)]
elif isinstance(usecols, numpy.ndarray):
usecols = usecols.tolist()
else:
usecols = list(usecols)
# create valid dtype object
if isinstance(dtype, (list, tuple)):
dtype = [dt if isinstance(dt, tuple) else ('', dt) for dt in dtype]
dtype = numpy.dtype(dtype)
# create list of dtypes to send to TextAdapter
if dtype.names is None:
# create list of homogenous scalar dtypes from single scalar dtype
numFields = len(usecols)
dtypes = [dtype] * numFields
fieldNames = None
else:
# create list of scalar dtypes from struct dtype
dtypes, fieldNames = unpack_dtype(dtype)
if fieldNames is not None:
list_names = ['' for x in range(adapter.field_count)]
for i, col in enumerate(usecols):
list_names[col] = fieldNames[i]
adapter.field_names = list_names
adapter.set_field_types(types=dict(zip(usecols, dtypes)))
if converters is not None:
for field, converter in converters.items():
adapter.set_converter(field, converter)
array = adapter[usecols][:]
if dtype.fields is not None and numpy.object_ not in [
dt[0] for dt in array.dtype.fields.values()
]:
array.dtype = dtype
elif dtype.fields is None:
array.dtype = dtype
if dtype.names is None:
if adapter.field_count == 0:
array.shape = (adapter.size, )
else:
array.shape = (adapter.size, len(usecols))
# Multicolumn data are returned with shape (1, N, M), i.e.
# (1, 1, M) for a single row - remove the singleton dimension there
if array.ndim == 3 and array.shape[:2] == (1, 1):
array.shape = (1, -1)
# Verify that the array has at least dimensions `ndmin`.
# Check correctness of the values of `ndmin`
if not ndmin in [0, 1, 2]:
raise ValueError('Illegal value of ndmin keyword: %s' % ndmin)
# Tweak the size and shape of the arrays - remove extraneous dimensions
if array.ndim > ndmin:
array = numpy.squeeze(array)
# and ensure we have the minimum number of dimensions asked for
# - has to be in this order for the odd case ndmin=1, array.squeeze().ndim=0
if array.ndim < ndmin:
if ndmin == 1:
array = numpy.atleast_1d(array)
elif ndmin == 2:
array = numpy.atleast_2d(array).T
if unpack:
if len(dtype) > 1:
# For structured arrays, return an array for each field.
return [array[field] for field in dtype.names]
else:
return array.T
else:
return array