def test_c_reader_pheno(self): snpdata1 = Pheno(self.currentFolder + "/examples/toydata.phe").read() self.assertEqual(np.float64, snpdata1.val.dtype) snpdata1.val[1,0] = np.NaN # Inject a missing value to test writing and reading missing values output = "tempdir/snpreader/toydata.phe" create_directory_if_necessary(output) Pheno.write(output, snpdata1) snpreader = Pheno(output) _fortesting_JustCheckExists().input(snpreader) s = str(snpreader) snpdata2 = snpreader.read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata2.val, decimal=10) snpdata1 = Pheno(self.currentFolder + "/examples/toydata.phe").read() import pysnptools.util.pheno as pstpheno dict = pstpheno.loadOnePhen(self.currentFolder + "/examples/toydata.phe",missing="") snpdata3 = Pheno(dict).read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata3.val, decimal=10) dict = pstpheno.loadOnePhen(self.currentFolder + "/examples/toydata.phe",missing="",vectorize=True) assert len(dict['vals'].shape)==1, "test 1-d array of values" snpdata3 = Pheno(dict).read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata3.val, decimal=10) snpdata4 = Pheno(None,iid_if_none=snpdata1.iid) assert (snpdata4.row == snpdata1.row).all() and snpdata4.col_count == 0 snpdata5 = Pheno(self.currentFolder + "/examples/toydata.id.phe").read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata5.val, decimal=10) snpdata6 = Pheno(self.currentFolder + "/examples/toydata.fid.phe").read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata6.val, decimal=10)
def test_snp_dist2(self): logging.info("in test_snp_dist2") snpreader = Bed(self.currentFolder + "/../examples/toydata.5chrom.bed", count_A1=False) snp2dist = snpreader.as_dist(max_weight=2) s = str(snp2dist) _fortesting_JustCheckExists().input(snp2dist)
def test_c_reader_dat(self): snpreader = Dat(self.currentFolder + "/examples/toydata.dat")[:, ::100] _fortesting_JustCheckExists().input(snpreader) snpdata1 = snpreader.read() self.assertEqual(np.float64, snpdata1.val.dtype) self.assertTrue( np.allclose(self.snps[:, ::100], snpdata1.val, rtol=1e-05, atol=1e-05)) snpdata1.val[ 1, 2] = np.NaN # Inject a missing value to test writing and reading missing values output = "tempdir/snpreader/toydata.dat" create_directory_if_necessary(output) Dat.write(output, snpdata1) snpdata2 = Dat(output).read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata2.val, decimal=10) snpdata3 = snpdata1[:, 0:0].read() #create snpdata with no sids output = "tempdir/snpreader/toydata3.dat" Dat.write(output, snpdata3) snpdata4 = Dat(output).read() assert snpdata3 == snpdata4
def test_dist_snp2(self): logging.info("in test_dist_snp2") distreader = DistNpz(self.currentFolder + "/../examples/toydata.dist.npz") dist2snp = distreader.as_snp(max_weight=33) s = str(dist2snp) _fortesting_JustCheckExists().input(dist2snp)
def cmktest_writes(self): #=================================== # Defining sub functions #=================================== def _oned_int(c): return range(c) def _oned_str(c): return [str(i).encode('ascii') for i in range(c)] def _twooned_int(c): return [[i] for i in range(c)] def _twooned_str(c): return [[str(i).encode('ascii')] for i in range(c)] def _twotwod_int(c): return [[i,i] for i in range(c)] def _twotwod_str(c): return [[str(i).encode('ascii'),b"hello"] for i in range(c)] def _none(c): return None def _zero(c): return np.empty([c,0]) #=================================== # Staring main function #=================================== logging.info("starting 'test_writes'") np.random.seed(0) output_template = "tempdir/pstreader/writes.{0}.{1}" create_directory_if_necessary(output_template.format(0,"npz")) i = 0 for row_count in [5,2,1,0]: for col_count in [4,2,1,0]: val = np.random.normal(.5,2,size=(row_count,col_count)) for row_or_col_gen in [_oned_int,_oned_str,_twooned_int,_twooned_str,_twotwod_int,_twotwod_str]: row = row_or_col_gen(row_count) col = row_or_col_gen(col_count) for prop_gen in [_oned_int,_oned_str,_twooned_int,_twooned_str,_twotwod_int,_twotwod_str,_none,_zero]: row_prop = prop_gen(row_count) col_prop = prop_gen(col_count) pstdata = PstData(row,col,val,row_prop,col_prop,str(i)) for the_class,suffix in [(PstNpz,"npz"),(PstHdf5,"hdf5")]: filename = output_template.format(i,suffix) logging.info(filename) i += 1 the_class.write(filename,pstdata) for subsetter in [None, sp.s_[::2,::3]]: reader = the_class(filename) _fortesting_JustCheckExists().input(reader) subreader = reader if subsetter is None else reader[subsetter[0],subsetter[1]] readdata = subreader.read(order='C') expected = pstdata if subsetter is None else pstdata[subsetter[0],subsetter[1]].read() assert np.array_equal(readdata.val,expected.val) assert np.array_equal(readdata.row,expected.row) assert np.array_equal(readdata.col,expected.col) assert np.array_equal(readdata.row_property,expected.row_property) assert np.array_equal(readdata.col_property,expected.col_property) try: os.remove(filename) except: pass logging.info("done with 'test_writes'")
def test_writes(self): #=================================== # Defining sub functions #=================================== def _oned_int(c): return list(range(c)) def _oned_str(c): return [str(i) for i in range(c)] def _twooned_int(c): return [[i] for i in range(c)] def _twooned_str(c): return [[str(i)] for i in range(c)] def _twotwod_int(c): return [[i,i] for i in range(c)] def _twotwod_str(c): return [[str(i),"hello"] for i in range(c)] def _none(c): return None def _zero(c): return np.empty([c,0]) #=================================== # Staring main function #=================================== logging.info("starting 'test_writes'") np.random.seed(0) output_template = "tempdir/pstreader/writes.{0}.{1}" create_directory_if_necessary(output_template.format(0,"npz")) i = 0 for row_count in [5,2,1,0]: for col_count in [4,2,1,0]: val = np.random.normal(.5,2,size=(row_count,col_count)) for row_or_col_gen in [_oned_int,_oned_str,_twooned_int,_twooned_str,_twotwod_int,_twotwod_str]: row = row_or_col_gen(row_count) col = row_or_col_gen(col_count) for prop_gen in [_oned_int,_oned_str,_twooned_int,_twooned_str,_twotwod_int,_twotwod_str,_none,_zero]: row_prop = prop_gen(row_count) col_prop = prop_gen(col_count) pstdata = PstData(row,col,val,row_prop,col_prop,str(i)) for the_class,suffix in [(PstNpz,"npz"),(PstHdf5,"hdf5")]: filename = output_template.format(i,suffix) logging.info(filename) i += 1 the_class.write(filename,pstdata) for subsetter in [None, sp.s_[::2,::3]]: reader = the_class(filename) _fortesting_JustCheckExists().input(reader) subreader = reader if subsetter is None else reader[subsetter[0],subsetter[1]] readdata = subreader.read(order='C') expected = pstdata if subsetter is None else pstdata[subsetter[0],subsetter[1]].read() assert np.array_equal(readdata.val,expected.val) assert np.array_equal(readdata.row,expected.row) assert np.array_equal(readdata.col,expected.col) assert np.array_equal(readdata.row_property,expected.row_property) assert np.array_equal(readdata.col_property,expected.col_property) try: os.remove(filename) except: pass logging.info("done with 'test_writes'")
def test_c_reader_dense(self): snpdata1 = self.snpdata[:,::100].read() snpdata1.val[1,2] = np.NaN # Inject a missing value to test writing and reading missing values output = "tempdir/snpreader/toydata.dense.txt" create_directory_if_necessary(output) Dense.write(output, snpdata1) snpreader = Dense(output) _fortesting_JustCheckExists().input(snpreader) snpdata2 = snpreader.read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata2.val, decimal=10)
def test_c_reader_pheno(self): snpdata1 = Pheno(self.currentFolder + "/examples/toydata.phe").read() self.assertEqual(np.float64, snpdata1.val.dtype) snpdata1.val[ 1, 0] = np.NaN # Inject a missing value to test writing and reading missing values output = "tempdir/snpreader/toydata.phe" create_directory_if_necessary(output) Pheno.write(output, snpdata1) snpreader = Pheno(output) _fortesting_JustCheckExists().input(snpreader) s = str(snpreader) snpdata2 = snpreader.read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata2.val, decimal=10) snpdata1 = Pheno(self.currentFolder + "/examples/toydata.phe").read() import pysnptools.util.pheno as pstpheno dict = pstpheno.loadOnePhen(self.currentFolder + "/examples/toydata.phe", missing="") snpdata3 = Pheno(dict).read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata3.val, decimal=10) dict = pstpheno.loadOnePhen(self.currentFolder + "/examples/toydata.phe", missing="", vectorize=True) assert len(dict['vals'].shape) == 1, "test 1-d array of values" snpdata3 = Pheno(dict).read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata3.val, decimal=10) snpdata4 = Pheno(None, iid_if_none=snpdata1.iid) assert (snpdata4.row == snpdata1.row).all() and snpdata4.col_count == 0 snpdata5 = Pheno(self.currentFolder + "/examples/toydata.id.phe").read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata5.val, decimal=10) snpdata6 = Pheno(self.currentFolder + "/examples/toydata.fid.phe").read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata6.val, decimal=10)
def test_c_reader_dense(self): snpdata1 = self.snpdata[:, ::100].read() snpdata1.val[ 1, 2] = np.NaN # Inject a missing value to test writing and reading missing values output = "tempdir/snpreader/toydata.dense.txt" create_directory_if_necessary(output) Dense.write(output, snpdata1) snpreader = Dense(output) _fortesting_JustCheckExists().input(snpreader) s = str(snpreader) snpdata2 = snpreader.read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata2.val, decimal=10)
def test_read(self): np.random.seed(0) row_property=np.array([[1.0,2,2.5],[3,4,4.5],[5,6,6.5]]) col_property=np.array([[1.0,2,2.5,1],[3,4,4.5,3]]) pstdata = PstData(row=np.array([[1.0,2],[3,4],[5,6]]), col=np.array([["A","a"],["B","b"]]), val = np.random.normal(.5,2,size=(3,2)), row_property=row_property, col_property=col_property, name="test_read") assert pstdata.row_to_index([np.array([3.0,4])])[0] == 1 assert pstdata.col_to_index([np.array(["A","a"])])[0] == 0 assert np.array_equal(pstdata[1:,:2].row_property,row_property[1:]) assert np.array_equal(pstdata[1:,:2].col_property,col_property[:2]) pstdata2 = pstdata[:2,:2].read() from pysnptools.kernelreader.test import _fortesting_JustCheckExists _fortesting_JustCheckExists().input(pstdata) _fortesting_JustCheckExists().input(pstdata2) np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata3 = pstdata[[],:].read() assert pstdata3.val.shape[0] == 0 and pstdata3.val.shape[1]==2 pstdata.val = pstdata.val.copy(order='F') pstdata2 = pstdata[:2,:2].read() np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata2 = pstdata[:2,:2].read(order='F') np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata2 = pstdata[:2,:2].read(order='A') np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype=None,order='C') np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype='float32',order='C') np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2].astype(dtype='float32'), decimal=10) pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype='float32',order=None) np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2].astype(dtype='float32'), decimal=10) pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype=None,order='F') np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata4 = pstdata[::,::].read(force_python_only=True) np.testing.assert_array_almost_equal(pstdata4.val, pstdata.val, decimal=10) logging.info("done with test")
def test_read(self): np.random.seed(0) row_property=np.array([[1.0,2,2.5],[3,4,4.5],[5,6,6.5]]) col_property=np.array([[1.0,2,2.5,1],[3,4,4.5,3]]) pstdata = PstData(row=np.array([[1.0,2],[3,4],[5,6]]), col=np.array([["A","a"],["B","b"]]), val = np.random.normal(.5,2,size=(3,2)), row_property=row_property, col_property=col_property, name="test_read") assert pstdata.row_to_index([np.array([3.0,4])])[0] == 1 assert pstdata.col_to_index([np.array(["A","a"])])[0] == 0 assert np.array_equal(pstdata[1:,:2].row_property,row_property[1:]) assert np.array_equal(pstdata[1:,:2].col_property,col_property[:2]) pstdata2 = pstdata[:2,:2].read() from pysnptools.kernelreader.test import _fortesting_JustCheckExists _fortesting_JustCheckExists().input(pstdata) _fortesting_JustCheckExists().input(pstdata2) np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata3 = pstdata[[],:].read() assert pstdata3.val.shape[0] == 0 and pstdata3.val.shape[1]==2 pstdata.val = pstdata.val.copy(order='F') pstdata2 = pstdata[:2,:2].read() np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata2 = pstdata[:2,:2].read(order='F') np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata2 = pstdata[:2,:2].read(order='A') np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype=None,order='C') np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype='float32',order='C') np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2].astype(dtype='float32'), decimal=10) pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype='float32',order=None) np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2].astype(dtype='float32'), decimal=10) pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype=None,order='F') np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10) pstdata4 = pstdata[::,::].read(force_python_only=True) np.testing.assert_array_almost_equal(pstdata4.val, pstdata.val, decimal=10) logging.info("done with test")
def test_c_reader_ped(self): if False: #Too slow for routine testing snpdata1 = Ped(self.currentFolder + "/examples/toydata.ped")[::25,::1000].read() self.assertEqual(np.float64, snpdata1.val.dtype) TestPySnpTools.assert_match_012_210(self.snpdata[::25,::1000].read(),snpdata1) else: snpdata1 = self.snpdata[::25,::1000].read() output = "tempdir/snpreader/toydata.ped" create_directory_if_necessary(output) snpdata1.val[1,2] = np.NaN # Inject a missing value to test writing and reading missing values Ped.write(output, snpdata1) snpreader = Ped(output) _fortesting_JustCheckExists().input(snpreader) s = str(snpreader) snpdata2 = snpreader.read() TestPySnpTools.assert_match_012_210(snpdata1,snpdata2)
def test_write_distnpz_f64cpp_5(self): distreader = DistNpz(self.currentFolder + "/../examples/toydata.dist.npz") _fortesting_JustCheckExists().input(distreader) iid_index = 5 logging.info("iid={0}".format(iid_index)) #if distreader.iid_count % 4 == 0: # divisible by 4 isn't a good test # distreader = distreader[0:-1,:] #assert distreader.iid_count % 4 != 0 distdata = distreader[0:iid_index,:].read(order='F',dtype=np.float64) if distdata.iid_count > 0: distdata.val[-1,0] = float("NAN") output = "tempdir/toydata.F64cpp.{0}.dist.npz".format(iid_index) create_directory_if_necessary(output) DistNpz.write(output, distdata ) #,force_python_only=True) snpdata2 = DistNpz(output).read() np.testing.assert_array_almost_equal(distdata.val, snpdata2.val, decimal=10)
def test_write_bed_f64cpp_5(self): snpreader = Bed(self.currentFolder + "/examples/toydata",count_A1=False) from pysnptools.kernelreader.test import _fortesting_JustCheckExists _fortesting_JustCheckExists().input(snpreader) iid_index = 5 logging.info("iid={0}".format(iid_index)) #if snpreader.iid_count % 4 == 0: # divisible by 4 isn't a good test # snpreader = snpreader[0:-1,:] #assert snpreader.iid_count % 4 != 0 snpdata = snpreader[0:iid_index,:].read(order='F',dtype=np.float64) if snpdata.iid_count > 0: snpdata.val[-1,0] = float("NAN") output = "tempdir/toydata.F64cpp.{0}".format(iid_index) create_directory_if_necessary(output) Bed.write(output, snpdata ,count_A1=False) #,force_python_only=True) snpdata2 = Bed(output,count_A1=False).read() np.testing.assert_array_almost_equal(snpdata.val, snpdata2.val, decimal=10)
def test_c_reader_dat(self): snpreader = Dat(self.currentFolder + "/examples/toydata.dat")[:,::100] _fortesting_JustCheckExists().input(snpreader) snpdata1 = snpreader.read() self.assertEqual(np.float64, snpdata1.val.dtype) self.assertTrue(np.allclose(self.snps[:,::100], snpdata1.val, rtol=1e-05, atol=1e-05)) snpdata1.val[1,2] = np.NaN # Inject a missing value to test writing and reading missing values output = "tempdir/snpreader/toydata.dat" create_directory_if_necessary(output) Dat.write(output,snpdata1) snpdata2 = Dat(output).read() np.testing.assert_array_almost_equal(snpdata1.val, snpdata2.val, decimal=10) snpdata3 = snpdata1[:,0:0].read() #create snpdata with no sids output = "tempdir/snpreader/toydata3.dat" Dat.write(output,snpdata3) snpdata4 = Dat(output).read() assert snpdata3 == snpdata4
def test_c_reader_ped(self): if False: #Too slow for routine testing snpdata1 = Ped(self.currentFolder + "/examples/toydata.ped")[::25, ::1000].read() self.assertEqual(np.float64, snpdata1.val.dtype) TestLoader.assert_match_012_210(self.snpdata[::25, ::1000].read(), snpdata1) else: snpdata1 = self.snpdata[::25, ::1000].read() output = "tempdir/snpreader/toydata.ped" create_directory_if_necessary(output) snpdata1.val[ 1, 2] = np.NaN # Inject a missing value to test writing and reading missing values Ped.write(output, snpdata1) snpreader = Ped(output) _fortesting_JustCheckExists().input(snpreader) s = str(snpreader) snpdata2 = snpreader.read() TestLoader.assert_match_012_210(snpdata1, snpdata2)
def test_write_bed_f64cpp_5(self): snpreader = Bed(self.currentFolder + "/examples/toydata", count_A1=False) from pysnptools.kernelreader.test import _fortesting_JustCheckExists _fortesting_JustCheckExists().input(snpreader) iid_index = 5 logging.info("iid={0}".format(iid_index)) #if snpreader.iid_count % 4 == 0: # divisible by 4 isn't a good test # snpreader = snpreader[0:-1,:] #assert snpreader.iid_count % 4 != 0 snpdata = snpreader[0:iid_index, :].read(order='F', dtype=np.float64) if snpdata.iid_count > 0: snpdata.val[-1, 0] = float("NAN") output = "tempdir/toydata.F64cpp.{0}".format(iid_index) create_directory_if_necessary(output) Bed.write(output, snpdata, count_A1=False) #,force_python_only=True) snpdata2 = Bed(output, count_A1=False).read() np.testing.assert_array_almost_equal(snpdata.val, snpdata2.val, decimal=10)
def test_writes(self): from pysnptools.distreader import DistData, DistHdf5, DistNpz, DistMemMap, Bgen from pysnptools.kernelreader.test import _fortesting_JustCheckExists the_class_and_suffix_list = [(DistNpz,"npz",None,None), (Bgen,"bgen",None,lambda filename,distdata: Bgen.write(filename,distdata,bits=32)), (DistHdf5,"hdf5",None,None), (DistMemMap,"memmap",None,None)] cant_do_col_prop_none_set = {'bgen'} cant_do_col_len_0_set = {'bgen'} cant_do_row_count_zero_set = {'bgen'} can_swap_0_2_set = {} can_change_col_names_set = {} ignore_fam_id_set = {} ignore_pos1_set = {'bgen'} ignore_pos_set = {} erase_any_write_dir = {} #=================================== # Starting main function #=================================== logging.info("starting 'test_writes'") np.random.seed(0) output_template = "tempdir/distreader/writes.{0}.{1}" create_directory_if_necessary(output_template.format(0,"npz")) i = 0 for row_count in [0,5,2,1]: for col_count in [4,2,1,0]: val=np.random.random(size=[row_count,col_count,3]) val /= val.sum(axis=2,keepdims=True) #make probabilities sum to 1 val[val==3]=np.NaN row = [('0','0'),('1','1'),('2','2'),('3','3'),('4','4')][:row_count] col = ['s0','s1','s2','s3','s4'][:col_count] for is_none in [True,False]: row_prop = None col_prop = None if is_none else [(x,x,x) for x in range(5)][:col_count] distdata = DistData(iid=row,sid=col,val=val,pos=col_prop,name=str(i)) for the_class,suffix,constructor,writer in the_class_and_suffix_list: constructor = constructor or (lambda filename: the_class(filename)) writer = writer or (lambda filename,distdata: the_class.write(filename,distdata)) if col_count == 0 and suffix in cant_do_col_len_0_set: continue if col_prop is None and suffix in cant_do_col_prop_none_set: continue if row_count==0 and suffix in cant_do_row_count_zero_set: continue filename = output_template.format(i,suffix) logging.info(filename) i += 1 if suffix in erase_any_write_dir and os.path.exists(filename): shutil.rmtree(filename) ret = writer(filename,distdata) assert ret is not None for subsetter in [None, np.s_[::2,::3]]: reader = constructor(filename) _fortesting_JustCheckExists().input(reader) subreader = reader if subsetter is None else reader[subsetter[0],subsetter[1]] readdata = subreader.read(order='C') expected = distdata if subsetter is None else distdata[subsetter[0],subsetter[1]].read() if not suffix in can_swap_0_2_set: assert np.allclose(readdata.val,expected.val,equal_nan=True) else: for col_index in range(readdata.col_count): assert (np.allclose(readdata.val[:,col_index],expected.val[:,col_index],equal_nan=True) or np.allclose(readdata.val[:,col_index]*-1+2,expected.val[:,col_index],equal_nan=True)) if not suffix in ignore_fam_id_set: assert np.array_equal(readdata.row,expected.row) else: assert np.array_equal(readdata.row[:,1],expected.row[:,1]) if not suffix in can_change_col_names_set: assert np.array_equal(readdata.col,expected.col) else: assert readdata.col_count==expected.col_count assert np.array_equal(readdata.row_property,expected.row_property) or (readdata.row_property.shape[1]==0 and expected.row_property.shape[1]==0) if suffix in ignore_pos1_set: assert np.allclose(readdata.col_property[:,[0,2]],expected.col_property[:,[0,2]],equal_nan=True) or (readdata.col_property.shape[1]==0 and expected.col_property.shape[1]==0) elif not suffix in ignore_pos_set: assert np.allclose(readdata.col_property,expected.col_property,equal_nan=True) or (readdata.col_property.shape[1]==0 and expected.col_property.shape[1]==0) else: assert len(readdata.col_property)==len(expected.col_property) try: os.remove(filename) except: pass logging.info("done with 'test_writes'")
def test_writes(self): #=================================== # Defining sub functions #=================================== def _oned_int(c): return range(c) def _oned_str(c): return [str(i).encode('ascii') for i in range(c)] def _twooned_int(c): return [[i] for i in range(c)] def _twooned_str(c): return [[str(i).encode('ascii')] for i in range(c)] def _twotwod_int(c): return [[i, i] for i in range(c)] def _twotwod_str(c): return [[str(i).encode('ascii'), b"hello"] for i in range(c)] #def _twotwod_U(c): # return [[str(i).encode('UTF-8'),u"hello"] for i in range(c)] def _none(c): return None def _zero(c): return np.empty([c, 0], dtype='S') #=================================== # Starting main function #=================================== logging.info("starting 'test_writes'") np.random.seed(0) temp_dir = tempfile.TemporaryDirectory("pstreader") output_template = temp_dir.name + '/writes.{0}.{1}' i = 0 for row_count in [5, 2, 1, 0]: for col_count in [4, 2, 1, 0]: for val_shape in [3, None, 1]: val = np.random.normal(.5, 2, size=( row_count, col_count)) if val_shape is None else np.random.normal( .5, 2, size=(row_count, col_count, val_shape)) for row_or_col_gen in [ _oned_int, _oned_str, _twooned_int, _twooned_str, _twotwod_int, _twotwod_str ]: #!!!,_twotwod_U can't roundtrop Unicode in hdf5 row = row_or_col_gen(row_count) col = row_or_col_gen(col_count) for prop_gen in [ _none, _oned_str, _oned_int, _twooned_int, _twooned_str, _twotwod_int, _twotwod_str, _zero ]: #!!!_twotwod_U can't round trip Unicode because Hdf5 doesn't like it. row_prop = prop_gen(row_count) col_prop = prop_gen(col_count) pstdata = PstData(row, col, val, row_prop, col_prop, str(i)) for the_class, suffix in [(PstMemMap, "memmap"), (PstHdf5, "hdf5"), (PstNpz, "npz")]: filename = output_template.format(i, suffix) logging.info(filename) i += 1 the_class.write(filename, pstdata) reader = the_class( filename ) if suffix != 'hdf5' else the_class( filename, block_size=3) _fortesting_JustCheckExists().input(reader) for subsetter in [None, np.s_[::2, ::3]]: subreader = reader if subsetter is None else reader[ subsetter[0], subsetter[1]] expected = pstdata if subsetter is None else pstdata[ subsetter[0], subsetter[1]].read() for order in ['C', 'F', 'A']: for force_python_only in [True, False]: readdata = subreader.read( order=order, force_python_only= force_python_only) assert np.array_equal( readdata.val, expected.val) assert np.array_equal( readdata.row, expected.row) assert np.array_equal( readdata.col, expected.col) assert np.array_equal( readdata.row_property, expected.row_property ) or ( readdata.row_property.shape[1] == 0 and expected.row_property.shape[1] == 0) assert np.array_equal( readdata.col_property, expected.col_property ) or ( readdata.col_property.shape[1] == 0 and expected.col_property.shape[1] == 0) if suffix in {'memmap', 'hdf5'}: reader.flush() os.remove(filename) temp_dir.cleanup() logging.info("done with 'test_writes'")