def test_usecols(self): a = np.array([[1, 2], [3, 4]], float) c = BytesIO() np.savetxt(c, a) c.seek(0) x = iopro.loadtxt(c, dtype=float, usecols=(1,)) assert_array_equal(x, a[:, 1]) a = np.array([[1, 2, 3], [3, 4, 5]], float) c = BytesIO() np.savetxt(c, a) c.seek(0) x = iopro.loadtxt(c, dtype=float, usecols=(1, 2)) assert_array_equal(x, a[:, 1:]) # Testing with arrays instead of tuples. c.seek(0) x = iopro.loadtxt(c, dtype=float, usecols=np.array([1, 2])) assert_array_equal(x, a[:, 1:]) # Checking with dtypes defined converters. data = '''JOE 70.1 25.3\nBOB 60.5 27.9''' c = StringIO(data) names = ['stid', 'temp'] dtypes = ['S3', 'f8'] arr = iopro.loadtxt(c, usecols=(0, 2), dtype=list(zip(names, dtypes))) assert_equal(arr['stid'], asbytes_nested(["JOE", "BOB"])) assert_equal(arr['temp'], [25.3, 27.9])
def test_array(self): c = StringIO() c.write('1 2\n3 4') c.seek(0) x = iopro.loadtxt(c, dtype=int) a = np.array([[1, 2], [3, 4]], int) assert_array_equal(x, a) c.seek(0) x = iopro.loadtxt(c, dtype=float) a = np.array([[1, 2], [3, 4]], float) assert_array_equal(x, a)
def test_unused_converter(self): assert_equal(True, False) c = StringIO() c.writelines(['1 21\n', '3 42\n']) c.seek(0) data = iopro.loadtxt(c, usecols=(1,), converters={0: lambda s: int(s, 16)}) assert_array_equal(data, [21, 42]) c.seek(0) data = iopro.loadtxt(c, usecols=(1,), converters={1: lambda s: int(s, 16)}) assert_array_equal(data, [33, 66])
def test_1D(self): c = StringIO() c.write('1\n2\n3\n4\n') c.seek(0) x = iopro.loadtxt(c, dtype=int) a = np.array([1, 2, 3, 4], int) assert_array_equal(x, a) c = StringIO() c.write('1,2,3,4\n') c.seek(0) x = iopro.loadtxt(c, dtype=int, delimiter=',') a = np.array([1, 2, 3, 4], int) assert_array_equal(x, a)
def test_empty_file(self): warn_ctx = WarningManager() warn_ctx.__enter__() try: warnings.filterwarnings("ignore", message="loadtxt: Empty input file:") c = StringIO() x = iopro.loadtxt(c) assert_equal(x.shape, (0,)) x = iopro.loadtxt(c, dtype=np.int64) assert_equal(x.shape, (0,)) assert_(x.dtype == np.int64) finally: warn_ctx.__exit__()
def test_int64_type(self): tgt = (-9223372036854775807, 9223372036854775807) c = StringIO() c.write("%s %s" % tgt) c.seek(0) res = iopro.loadtxt(c, dtype=np.int64) assert_equal(res, tgt)
def test_uint64_type(self): tgt = (9223372043271415339, 9223372043271415853) c = StringIO() c.write("%s %s" % tgt) c.seek(0) res = iopro.loadtxt(c, dtype=np.uint64) assert_equal(res, tgt)
def test_skiprows(self): c = StringIO() c.write('comment\n1,2,3,5\n') c.seek(0) x = iopro.loadtxt(c, dtype=int, delimiter=',', \ skiprows=1) a = np.array([1, 2, 3, 5], int) assert_array_equal(x, a) c = StringIO() c.write('# comment\n1,2,3,5\n') c.seek(0) x = iopro.loadtxt(c, dtype=int, delimiter=',', \ skiprows=1) a = np.array([1, 2, 3, 5], int) assert_array_equal(x, a)
def test_generator_source(self): def count(): for i in range(10): yield "%d" % i res = iopro.loadtxt(count()) assert_array_equal(res, np.arange(10))
def test_skiprows(self): "Test row skipping" control = np.array([1, 2, 3, 5], int) kwargs = dict(dtype=int, delimiter=',') # data = StringIO('# comment\n1,2,3,5\n') test = iopro.loadtxt(data, skiprows=1, **kwargs) assert_equal(test, control)
def test_3d_shaped_dtype(self): c = StringIO("aaaa 1.0 8.0 1 2 3 4 5 6 7 8 9 10 11 12") dt = np.dtype([('name', 'S4'), ('x', float), ('y', float), ('block', int, (2, 2, 3))]) x = iopro.loadtxt(c, dtype=dt) a = np.array([('aaaa', 1.0, 8.0, [[[1, 2, 3], [4, 5, 6]],[[7, 8, 9], [10, 11, 12]]])], dtype=dt) assert_array_equal(x, a)
def test_fancy_dtype(self): c = StringIO() c.write('1,2,3.0\n4,5,6.0\n') c.seek(0) dt = np.dtype([('x', int), ('y', [('t', int), ('s', float)])]) x = iopro.loadtxt(c, dtype=dt, delimiter=',') a = np.array([(1, (2, 3.0)), (4, (5, 6.0))], dt) assert_array_equal(x, a)
def test_missing(self): c = StringIO() c.write('1,2,3,,5\n') c.seek(0) x = iopro.loadtxt(c, dtype=int, delimiter=',', \ converters={3:lambda s: int(s or - 999)}) a = np.array([1, 2, 3, -999, 5], int) assert_array_equal(x, a)
def test_converters_with_usecols(self): c = StringIO() c.write('1,2,3,,5\n6,7,8,9,10\n') c.seek(0) x = iopro.loadtxt(c, dtype=int, delimiter=',', \ converters={3:lambda s: int(s or - 999)}, \ usecols=(1, 3,)) a = np.array([[2, -999], [7, 9]], int) assert_array_equal(x, a)
def test_empty_field_after_tab(self): c = StringIO() c.write('1 \t2 \t3\tstart \n4\t5\t6\t \n7\t8\t9.5\t') c.seek(0) dt = { 'names': ('x', 'y', 'z', 'comment'), 'formats': ('<i4', '<i4', '<f4', '|S8')} x = iopro.loadtxt(c, dtype=dt, delimiter='\t') a = np.array(['start ', ' ', ''], dtype='|S8') assert_array_equal(x['comment'], a)
def test_record(self): c = StringIO() c.write('1 2\n3 4') c.seek(0) x = iopro.loadtxt(c, dtype=[('x', np.int32), ('y', np.int32)]) a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) assert_array_equal(x, a) d = StringIO() d.write('M 64.0 75.0\nF 25.0 60.0') d.seek(0) mydescriptor = {'names': ('gender', 'age', 'weight'), 'formats': ('S1', 'i4', 'f4')} b = np.array([('M', 64.0, 75.0), ('F', 25.0, 60.0)], dtype=mydescriptor) y = iopro.loadtxt(d, dtype=mydescriptor) assert_array_equal(y, b)
def test_gzip_loadtxt_from_string(): s = StringIO() f = gzip.GzipFile(fileobj=s, mode="w") f.write(asbytes('1 2 3\n')) f.close() s.seek(0) f = gzip.GzipFile(fileobj=s, mode="r") assert_array_equal(iopro.loadtxt(f), [1, 2, 3])
def test_universal_newline(self): f, name = mkstemp() os.write(f, b'1 21\r3 42\r') os.close(f) try: data = iopro.loadtxt(name) assert_array_equal(data, [[1, 21], [3, 42]]) finally: os.unlink(name)
def test_structure_unpack(self): txt = StringIO("M 21 72\nF 35 58") dt = { 'names': ('a', 'b', 'c'), 'formats': ('|S1', '<i4', '<f4')} a, b, c = iopro.loadtxt(txt, dtype=dt, unpack=True) assert_(a.dtype.str == '|S1') assert_(b.dtype.str == '<i4') assert_(c.dtype.str == '<f4') assert_array_equal(a, np.array(['M', 'F'], dtype='|S1')) assert_array_equal(b, np.array([21, 35], dtype='<i4')) assert_array_equal(c, np.array([ 72., 58.], dtype='<f4'))
def test_array(self): "Test outputing a standard ndarray" data = BytesIO(b'1 2\n3 4') control = np.array([[1, 2], [3, 4]], dtype=int) test = np.ndfromtxt(data, dtype=int) assert_array_equal(test, control) # data.seek(0) control = np.array([[1, 2], [3, 4]], dtype=float) test = iopro.loadtxt(data, dtype=float) assert_array_equal(test, control)
def test_ndmin_keyword(self): c = StringIO() c.write('1,2,3\n4,5,6') c.seek(0) assert_raises(iopro.DataTypeError, iopro.loadtxt, c, ndmin=3) c.seek(0) assert_raises(iopro.DataTypeError, iopro.loadtxt, c, ndmin=1.5) c.seek(0) x = iopro.loadtxt(c, dtype=int, delimiter=',', ndmin=1) a = np.array([[1, 2, 3], [4, 5, 6]]) assert_array_equal(x, a) d = StringIO() d.write('0,1,2') d.seek(0) x = iopro.loadtxt(d, dtype=int, delimiter=',', ndmin=2) assert_(x.shape == (1, 3)) d.seek(0) x = iopro.loadtxt(d, dtype=int, delimiter=',', ndmin=1) assert_(x.shape == (3,)) d.seek(0) x = iopro.loadtxt(d, dtype=int, delimiter=',', ndmin=0) assert_(x.shape == (3,)) e = StringIO() e.write('0\n1\n2') e.seek(0) x = iopro.loadtxt(e, dtype=int, delimiter=',', ndmin=2) assert_(x.shape == (3, 1)) e.seek(0) x = iopro.loadtxt(e, dtype=int, delimiter=',', ndmin=1) assert_(x.shape == (3,)) e.seek(0) x = iopro.loadtxt(e, dtype=int, delimiter=',', ndmin=0) assert_(x.shape == (3,)) # Test ndmin kw with empty file. warn_ctx = WarningManager() warn_ctx.__enter__() try: warnings.filterwarnings("ignore", message="loadtxt: Empty input file:") f = StringIO() assert_(iopro.loadtxt(f, ndmin=2).shape == (0, 1,)) assert_(iopro.loadtxt(f, ndmin=1).shape == (0,)) finally: warn_ctx.__exit__()
def test_dtype_with_object(self): assert_equal(True, False) "Test using an explicit dtype with an object" from datetime import date import time data = """ 1; 2001-01-01 2; 2002-01-31 """ ndtype = [('idx', int), ('code', np.object)] func = lambda s: strptime(s.strip(), "%Y-%m-%d") converters = {1: func} test = iopro.loadtxt(StringIO(data), delimiter=";", dtype=ndtype, converters=converters) control = np.array([(1, datetime(2001, 1, 1)), (2, datetime(2002, 1, 31))], dtype=ndtype) assert_equal(test, control)
def InitSidechain_hdf(self): print "Initializing Sidechain data..." if not query_yes_no("Are you sure you want to re-write {}?".format(self.sc_h5file), default="no"): print "File rewrite skipped." return Nsites = self.Nsites # Load all of the CSV file paths into an array with open(self.csv_files) as f: csv_files = f.readlines() print("Loading data from {} csv files...".format(len(csv_files))) dset_tags = [self.time_h5tag + "%d" % i for i in xrange(1,Nsites+1)] # Load each CSV sequentially into an HDF file that is created on the first step first_file=True for i,file in enumerate(csv_files): print "File #{}: {}".format(i, file.strip()) x = iopro.loadtxt(file.strip(),delimiter=',') assert(format(x[0::Nsites].shape == x[(Nsites-1)::Nsites].shape)) assert(len(x) % Nsites == 0) print "\tX total shape: {}".format((x.shape[0]/Nsites, Nsites, len(x[0,:]))) Ntimes = len(x) / Nsites # Create the HDF file if we are looping through for the first time using the sizes from the csv files if first_file: Ncoarse = len(x[0]) print "\tCreating new datasets, loaded file with Ncoarse = {} and Ntimes = {}".format(Ncoarse, Ntimes) dEhdf5_init(self.sc_h5file, None, 'w') for dset_tag in dset_tags: dEhdf5_init(self.sc_h5file, dset_tag, 'a', Ncoarse=Ncoarse, dt_ps=.005) first_file=False h5_out = h5py.File(self.sc_h5file,'a') try: dsets = [h5_out[dset_tag] for dset_tag in dset_tags] for i,ds in enumerate(dsets): oldlen = ds.shape[0] newlen = oldlen + Ntimes ds.resize(newlen, axis=0) ds[oldlen:newlen,:] = x[i::Nsites,:] h5_out.close() except: print "Write failed..." h5_out.close() print("Error: {}".format(sys.exc_info()[0])) raise
def test_gzip_loadtxt(): # Thanks to another windows brokeness, we can't use # NamedTemporaryFile: a file created from this function cannot be # reopened by another open call. So we first put the gzipped string # of the test reference array, write it to a securely opened file, # which is then read from by the loadtxt function s = BytesIO() g = gzip.GzipFile(fileobj=s, mode='wb') g.write(asbytes('1 2 3\n')) g.close() s.seek(0) f, name = mkstemp(suffix='.gz') try: os.write(f, s.read()) s.close() assert_array_equal(iopro.loadtxt(name), [1, 2, 3]) finally: os.close(f) os.unlink(name)