def test_usecols(self):
a = np.array([[1, 2], [3, 4]], float)
c = BytesIO()
np.savetxt(c, a)
c.seek(0)
x = iopro.loadtxt(c, dtype=float, usecols=(1,))
assert_array_equal(x, a[:, 1])
a = np.array([[1, 2, 3], [3, 4, 5]], float)
c = BytesIO()
np.savetxt(c, a)
c.seek(0)
x = iopro.loadtxt(c, dtype=float, usecols=(1, 2))
assert_array_equal(x, a[:, 1:])
# Testing with arrays instead of tuples.
c.seek(0)
x = iopro.loadtxt(c, dtype=float, usecols=np.array([1, 2]))
assert_array_equal(x, a[:, 1:])
# Checking with dtypes defined converters.
data = '''JOE 70.1 25.3\nBOB 60.5 27.9'''
c = StringIO(data)
names = ['stid', 'temp']
dtypes = ['S3', 'f8']
arr = iopro.loadtxt(c, usecols=(0, 2), dtype=list(zip(names, dtypes)))
assert_equal(arr['stid'], asbytes_nested(["JOE", "BOB"]))
assert_equal(arr['temp'], [25.3, 27.9])
def test_array(self):
c = StringIO()
c.write('1 2\n3 4')
c.seek(0)
x = iopro.loadtxt(c, dtype=int)
a = np.array([[1, 2], [3, 4]], int)
assert_array_equal(x, a)
c.seek(0)
x = iopro.loadtxt(c, dtype=float)
a = np.array([[1, 2], [3, 4]], float)
assert_array_equal(x, a)
def test_unused_converter(self):
assert_equal(True, False)
c = StringIO()
c.writelines(['1 21\n', '3 42\n'])
c.seek(0)
data = iopro.loadtxt(c, usecols=(1,),
converters={0: lambda s: int(s, 16)})
assert_array_equal(data, [21, 42])
c.seek(0)
data = iopro.loadtxt(c, usecols=(1,),
converters={1: lambda s: int(s, 16)})
assert_array_equal(data, [33, 66])
def test_1D(self):
c = StringIO()
c.write('1\n2\n3\n4\n')
c.seek(0)
x = iopro.loadtxt(c, dtype=int)
a = np.array([1, 2, 3, 4], int)
assert_array_equal(x, a)
c = StringIO()
c.write('1,2,3,4\n')
c.seek(0)
x = iopro.loadtxt(c, dtype=int, delimiter=',')
a = np.array([1, 2, 3, 4], int)
assert_array_equal(x, a)
def test_empty_file(self):
warn_ctx = WarningManager()
warn_ctx.__enter__()
try:
warnings.filterwarnings("ignore",
message="loadtxt: Empty input file:")
c = StringIO()
x = iopro.loadtxt(c)
assert_equal(x.shape, (0,))
x = iopro.loadtxt(c, dtype=np.int64)
assert_equal(x.shape, (0,))
assert_(x.dtype == np.int64)
finally:
warn_ctx.__exit__()
def test_int64_type(self):
tgt = (-9223372036854775807, 9223372036854775807)
c = StringIO()
c.write("%s %s" % tgt)
c.seek(0)
res = iopro.loadtxt(c, dtype=np.int64)
assert_equal(res, tgt)
def test_uint64_type(self):
tgt = (9223372043271415339, 9223372043271415853)
c = StringIO()
c.write("%s %s" % tgt)
c.seek(0)
res = iopro.loadtxt(c, dtype=np.uint64)
assert_equal(res, tgt)
def test_skiprows(self):
c = StringIO()
c.write('comment\n1,2,3,5\n')
c.seek(0)
x = iopro.loadtxt(c, dtype=int, delimiter=',', \
skiprows=1)
a = np.array([1, 2, 3, 5], int)
assert_array_equal(x, a)
c = StringIO()
c.write('# comment\n1,2,3,5\n')
c.seek(0)
x = iopro.loadtxt(c, dtype=int, delimiter=',', \
skiprows=1)
a = np.array([1, 2, 3, 5], int)
assert_array_equal(x, a)
def test_generator_source(self):
def count():
for i in range(10):
yield "%d" % i
res = iopro.loadtxt(count())
assert_array_equal(res, np.arange(10))
def test_skiprows(self):
"Test row skipping"
control = np.array([1, 2, 3, 5], int)
kwargs = dict(dtype=int, delimiter=',')
#
data = StringIO('# comment\n1,2,3,5\n')
test = iopro.loadtxt(data, skiprows=1, **kwargs)
assert_equal(test, control)
def test_3d_shaped_dtype(self):
c = StringIO("aaaa 1.0 8.0 1 2 3 4 5 6 7 8 9 10 11 12")
dt = np.dtype([('name', 'S4'), ('x', float), ('y', float),
('block', int, (2, 2, 3))])
x = iopro.loadtxt(c, dtype=dt)
a = np.array([('aaaa', 1.0, 8.0, [[[1, 2, 3], [4, 5, 6]],[[7, 8, 9], [10, 11, 12]]])],
dtype=dt)
assert_array_equal(x, a)
def test_fancy_dtype(self):
c = StringIO()
c.write('1,2,3.0\n4,5,6.0\n')
c.seek(0)
dt = np.dtype([('x', int), ('y', [('t', int), ('s', float)])])
x = iopro.loadtxt(c, dtype=dt, delimiter=',')
a = np.array([(1, (2, 3.0)), (4, (5, 6.0))], dt)
assert_array_equal(x, a)
def test_missing(self):
c = StringIO()
c.write('1,2,3,,5\n')
c.seek(0)
x = iopro.loadtxt(c, dtype=int, delimiter=',', \
converters={3:lambda s: int(s or - 999)})
a = np.array([1, 2, 3, -999, 5], int)
assert_array_equal(x, a)
def test_converters_with_usecols(self):
c = StringIO()
c.write('1,2,3,,5\n6,7,8,9,10\n')
c.seek(0)
x = iopro.loadtxt(c, dtype=int, delimiter=',', \
converters={3:lambda s: int(s or - 999)}, \
usecols=(1, 3,))
a = np.array([[2, -999], [7, 9]], int)
assert_array_equal(x, a)
def test_empty_field_after_tab(self):
c = StringIO()
c.write('1 \t2 \t3\tstart \n4\t5\t6\t \n7\t8\t9.5\t')
c.seek(0)
dt = { 'names': ('x', 'y', 'z', 'comment'),
'formats': ('<i4', '<i4', '<f4', '|S8')}
x = iopro.loadtxt(c, dtype=dt, delimiter='\t')
a = np.array(['start ', ' ', ''], dtype='|S8')
assert_array_equal(x['comment'], a)
def test_record(self):
c = StringIO()
c.write('1 2\n3 4')
c.seek(0)
x = iopro.loadtxt(c, dtype=[('x', np.int32), ('y', np.int32)])
a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')])
assert_array_equal(x, a)
d = StringIO()
d.write('M 64.0 75.0\nF 25.0 60.0')
d.seek(0)
mydescriptor = {'names': ('gender', 'age', 'weight'),
'formats': ('S1',
'i4', 'f4')}
b = np.array([('M', 64.0, 75.0),
('F', 25.0, 60.0)], dtype=mydescriptor)
y = iopro.loadtxt(d, dtype=mydescriptor)
assert_array_equal(y, b)
def test_gzip_loadtxt_from_string():
s = StringIO()
f = gzip.GzipFile(fileobj=s, mode="w")
f.write(asbytes('1 2 3\n'))
f.close()
s.seek(0)
f = gzip.GzipFile(fileobj=s, mode="r")
assert_array_equal(iopro.loadtxt(f), [1, 2, 3])
def test_universal_newline(self):
f, name = mkstemp()
os.write(f, b'1 21\r3 42\r')
os.close(f)
try:
data = iopro.loadtxt(name)
assert_array_equal(data, [[1, 21], [3, 42]])
finally:
os.unlink(name)
def test_structure_unpack(self):
txt = StringIO("M 21 72\nF 35 58")
dt = { 'names': ('a', 'b', 'c'), 'formats': ('|S1', '<i4', '<f4')}
a, b, c = iopro.loadtxt(txt, dtype=dt, unpack=True)
assert_(a.dtype.str == '|S1')
assert_(b.dtype.str == '<i4')
assert_(c.dtype.str == '<f4')
assert_array_equal(a, np.array(['M', 'F'], dtype='|S1'))
assert_array_equal(b, np.array([21, 35], dtype='<i4'))
assert_array_equal(c, np.array([ 72., 58.], dtype='<f4'))
def test_array(self):
"Test outputing a standard ndarray"
data = BytesIO(b'1 2\n3 4')
control = np.array([[1, 2], [3, 4]], dtype=int)
test = np.ndfromtxt(data, dtype=int)
assert_array_equal(test, control)
#
data.seek(0)
control = np.array([[1, 2], [3, 4]], dtype=float)
test = iopro.loadtxt(data, dtype=float)
assert_array_equal(test, control)
def test_ndmin_keyword(self):
c = StringIO()
c.write('1,2,3\n4,5,6')
c.seek(0)
assert_raises(iopro.DataTypeError, iopro.loadtxt, c, ndmin=3)
c.seek(0)
assert_raises(iopro.DataTypeError, iopro.loadtxt, c, ndmin=1.5)
c.seek(0)
x = iopro.loadtxt(c, dtype=int, delimiter=',', ndmin=1)
a = np.array([[1, 2, 3], [4, 5, 6]])
assert_array_equal(x, a)
d = StringIO()
d.write('0,1,2')
d.seek(0)
x = iopro.loadtxt(d, dtype=int, delimiter=',', ndmin=2)
assert_(x.shape == (1, 3))
d.seek(0)
x = iopro.loadtxt(d, dtype=int, delimiter=',', ndmin=1)
assert_(x.shape == (3,))
d.seek(0)
x = iopro.loadtxt(d, dtype=int, delimiter=',', ndmin=0)
assert_(x.shape == (3,))
e = StringIO()
e.write('0\n1\n2')
e.seek(0)
x = iopro.loadtxt(e, dtype=int, delimiter=',', ndmin=2)
assert_(x.shape == (3, 1))
e.seek(0)
x = iopro.loadtxt(e, dtype=int, delimiter=',', ndmin=1)
assert_(x.shape == (3,))
e.seek(0)
x = iopro.loadtxt(e, dtype=int, delimiter=',', ndmin=0)
assert_(x.shape == (3,))
# Test ndmin kw with empty file.
warn_ctx = WarningManager()
warn_ctx.__enter__()
try:
warnings.filterwarnings("ignore",
message="loadtxt: Empty input file:")
f = StringIO()
assert_(iopro.loadtxt(f, ndmin=2).shape == (0, 1,))
assert_(iopro.loadtxt(f, ndmin=1).shape == (0,))
finally:
warn_ctx.__exit__()
def test_dtype_with_object(self):
assert_equal(True, False)
"Test using an explicit dtype with an object"
from datetime import date
import time
data = """ 1; 2001-01-01
2; 2002-01-31 """
ndtype = [('idx', int), ('code', np.object)]
func = lambda s: strptime(s.strip(), "%Y-%m-%d")
converters = {1: func}
test = iopro.loadtxt(StringIO(data), delimiter=";", dtype=ndtype,
converters=converters)
control = np.array([(1, datetime(2001, 1, 1)), (2, datetime(2002, 1, 31))],
dtype=ndtype)
assert_equal(test, control)
def InitSidechain_hdf(self):
print "Initializing Sidechain data..."
if not query_yes_no("Are you sure you want to re-write {}?".format(self.sc_h5file), default="no"):
print "File rewrite skipped."
return
Nsites = self.Nsites
# Load all of the CSV file paths into an array
with open(self.csv_files) as f:
csv_files = f.readlines()
print("Loading data from {} csv files...".format(len(csv_files)))
dset_tags = [self.time_h5tag + "%d" % i for i in xrange(1,Nsites+1)]
# Load each CSV sequentially into an HDF file that is created on the first step
first_file=True
for i,file in enumerate(csv_files):
print "File #{}: {}".format(i, file.strip())
x = iopro.loadtxt(file.strip(),delimiter=',')
assert(format(x[0::Nsites].shape == x[(Nsites-1)::Nsites].shape))
assert(len(x) % Nsites == 0)
print "\tX total shape: {}".format((x.shape[0]/Nsites, Nsites, len(x[0,:])))
Ntimes = len(x) / Nsites
# Create the HDF file if we are looping through for the first time using the sizes from the csv files
if first_file:
Ncoarse = len(x[0])
print "\tCreating new datasets, loaded file with Ncoarse = {} and Ntimes = {}".format(Ncoarse, Ntimes)
dEhdf5_init(self.sc_h5file, None, 'w')
for dset_tag in dset_tags:
dEhdf5_init(self.sc_h5file, dset_tag, 'a', Ncoarse=Ncoarse, dt_ps=.005)
first_file=False
h5_out = h5py.File(self.sc_h5file,'a')
try:
dsets = [h5_out[dset_tag] for dset_tag in dset_tags]
for i,ds in enumerate(dsets):
oldlen = ds.shape[0]
newlen = oldlen + Ntimes
ds.resize(newlen, axis=0)
ds[oldlen:newlen,:] = x[i::Nsites,:]
h5_out.close()
except:
print "Write failed..."
h5_out.close()
print("Error: {}".format(sys.exc_info()[0]))
raise
def test_gzip_loadtxt():
# Thanks to another windows brokeness, we can't use
# NamedTemporaryFile: a file created from this function cannot be
# reopened by another open call. So we first put the gzipped string
# of the test reference array, write it to a securely opened file,
# which is then read from by the loadtxt function
s = BytesIO()
g = gzip.GzipFile(fileobj=s, mode='wb')
g.write(asbytes('1 2 3\n'))
g.close()
s.seek(0)
f, name = mkstemp(suffix='.gz')
try:
os.write(f, s.read())
s.close()
assert_array_equal(iopro.loadtxt(name), [1, 2, 3])
finally:
os.close(f)
os.unlink(name)
