def __init__(self, iid, test=None): #!!! add docs and test for test
super(Identity, self).__init__()
if test is None:
test = iid
if test is iid:
iid = PstData._fixup_input(
iid,
empty_creator=lambda ignore: np.empty([0, 2], dtype='str'),
dtype='str')
test = iid
else:
iid = PstData._fixup_input(
iid,
empty_creator=lambda ignore: np.empty([0, 2], dtype='str'),
dtype='str')
test = PstData._fixup_input(
test,
empty_creator=lambda ignore: np.empty([0, 2], dtype='str'),
dtype='str')
if len(iid) > 0:
self._row0 = iid
else:
self._row0 = self._empty
if len(test) > 0:
self._row1 = test
else:
self._row1 = self._empty
def __init__(self,
filename,
count_A1=None,
iid=None,
sid=None,
pos=None,
skip_format_check=False
): #!!!document these new optionals. they are here
super(Bed, self).__init__()
self._ran_once = False
self._file_pointer = None
self.filename = filename
if count_A1 is None:
warnings.warn(
"'count_A1' was not set. For now it will default to 'False', but in the future it will default to 'True'",
FutureWarning)
count_A1 = False
self.count_A1 = count_A1
self.skip_format_check = skip_format_check
if iid is not None:
self._row = PstData._fixup_input(
iid, empty_creator=lambda ignore: np.empty([0, 2], dtype=str))
if sid is not None:
self._col = PstData._fixup_input(
sid, empty_creator=lambda ignore: np.empty([0], dtype=str))
if pos is not None:
self._col_property = PstData._fixup_input(
pos,
count=len(self._col),
empty_creator=lambda count: np.array([[np.nan, np.nan, np.nan]]
* count))
def cmktest_repr_test(self):
np.random.seed(0)
row_property=np.array([[1.0,2,2.5],[3,4,4.5],[5,6,6.5]])
col_property=np.array([[1.0,2,2.5,1],[3,4,4.5,3]])
pstdata = PstData(row=np.array([[1.0,2],[3,4],[5,6]]),
col=np.array([("A","a"),("B","b")]),
val = np.random.normal(.5,2,size=(3,2)),
row_property=row_property,
col_property=col_property)
assert pstdata.col_to_index([("B","b")])[0] == 1
s = str(pstdata)
def _run_once(self):
if (self._ran_once):
return
row, col, val, row_property, col_property = self._run_once_inner()
PstData.__init__(self,
row,
col,
val,
row_property,
col_property,
name="np.memmap('{0}')".format(self._filename))
def __init__(self, iid, sid, val, pos=None, name=None, parent_string=None, copyinputs_function=None):
if parent_string is not None:
warnings.warn("'parent_string' is deprecated. Use 'name'", DeprecationWarning)
self._row = PstData._fixup_input(iid,empty_creator=lambda ignore:np.empty([0,2],dtype=str))
self._col = PstData._fixup_input(sid,empty_creator=lambda ignore:np.empty([0],dtype=str))
self._row_property = PstData._fixup_input(None,count=len(self._row),empty_creator=lambda count:np.empty([count,0],dtype=str))
self._col_property = PstData._fixup_input(pos,count=len(self._col),empty_creator=lambda count:np.array([[np.nan, np.nan, np.nan]]*count))
self.val = PstData._fixup_input_val(val,row_count=len(self._row),col_count=len(self._col),empty_creator=lambda row_count,col_count:np.empty([row_count,col_count],dtype=np.float64))
self._assert_iid_sid_pos()
self._name = name or parent_string or ""
self._std_string_list = []
def test_repr_test(self):
np.random.seed(0)
row_property=np.array([[1.0,2,2.5],[3,4,4.5],[5,6,6.5]])
col_property=np.array([[1.0,2,2.5,1],[3,4,4.5,3]])
pstdata = PstData(row=np.array([[1.0,2],[3,4],[5,6]]),
col=np.array([("A","a"),("B","b")]),
val = np.random.normal(.5,2,size=(3,2)),
row_property=row_property,
col_property=col_property)
assert pstdata.col_to_index([("B","b")])[0] == 1
s = str(pstdata)
def __init__(self, iid, sid, val, pos=None, name=None, copyinputs_function=None):
#We don't have a 'super(DistData, self).__init__()' here because DistData takes full responsibility for initializing both its superclasses
self._val = None
self._row = PstData._fixup_input(iid,empty_creator=lambda ignore:np.empty([0,2],dtype='str'),dtype='str')
self._col = PstData._fixup_input(sid,empty_creator=lambda ignore:np.empty([0],dtype='str'),dtype='str')
self._row_property = PstData._fixup_input(None,count=len(self._row),empty_creator=lambda count:np.empty([count,0],dtype='str'),dtype='str')
self._col_property = PstData._fixup_input(pos,count=len(self._col),empty_creator=lambda count:np.full([count, 3], np.nan))
self._val = PstData._fixup_input_val(val,row_count=len(self._row),col_count=len(self._col),empty_creator=lambda row_count,col_count:np.empty([row_count,col_count,3],dtype=np.float64))#!!!Replace empty with my FillNA method?
self._assert_iid_sid_pos(check_val=True)
self._name = name or ""
self._std_string_list = []
def test_inputs3(self):
from pysnptools.pstreader import PstData
np.random.seed(0)
row_property=None
col_property=None
pstdata = PstData(row=[[1.0,2.0],[3,4],[6,7]],
col=np.array([]),
val = [[],[],[]],
row_property=row_property,
col_property=col_property,
name="test_read")
assert pstdata.row_to_index([[3,4]])[0] == 1
assert np.array_equal(pstdata[1:,:2].row_property,pstdata.row_property[1:])
assert np.array_equal(pstdata[1:,:2].col_property,pstdata.col_property[:2])
logging.info("done with test")
def __init__(self, iid, sid, val, pos=None, name=None, parent_string=None, copyinputs_function=None):
#We don't have a 'super(SnpData, self).__init__()' here because SnpData takes full responsiblity for initializing both its superclasses
self.val = None
if parent_string is not None:
warnings.warn("'parent_string' is deprecated. Use 'name'", DeprecationWarning)
self._row = PstData._fixup_input(iid,empty_creator=lambda ignore:np.empty([0,2],dtype=str))
self._col = PstData._fixup_input(sid,empty_creator=lambda ignore:np.empty([0],dtype=str))
self._row_property = PstData._fixup_input(None,count=len(self._row),empty_creator=lambda count:np.empty([count,0],dtype=str))
self._col_property = PstData._fixup_input(pos,count=len(self._col),empty_creator=lambda count:np.array([[np.nan, np.nan, np.nan]]*count))
self.val = PstData._fixup_input_val(val,row_count=len(self._row),col_count=len(self._col),empty_creator=lambda row_count,col_count:np.empty([row_count,col_count],dtype=np.float64))
self._assert_iid_sid_pos()
self._name = name or parent_string or ""
self._std_string_list = []
def assert_approx_equal(distdata0, distdata1, atol):
from pysnptools.pstreader import PstData
assert PstData._allclose(distdata0.row, distdata1.row, equal_nan=True)
assert PstData._allclose(distdata0.col, distdata1.col, equal_nan=True)
assert PstData._allclose(distdata0.row_property,
distdata1.row_property,
equal_nan=True)
assert PstData._allclose(distdata0.col_property,
distdata1.col_property,
equal_nan=True)
np.testing.assert_allclose(distdata0.val,
distdata1.val,
atol=atol,
equal_nan=True,
verbose=True)
def test_every_read(self):
for order_from in ['F', 'C']:
for order_to in ['F', 'C']:
for dtype_from in [np.float32, np.float64]:
for dtype_to in [np.float32, np.float64]:
for val_shape in [None, 1, 3]:
for force_python_only in [True, False]:
np.random.seed(0)
val0 = np.random.normal(.5, 2, size=(
3, 2
)) if val_shape is None else np.random.normal(
.5, 2, size=(3, 2, val_shape))
val = np.array(val0,
order=order_from,
dtype=dtype_from)
pstdata = PstData(val=val,
row=list(range(3)),
col=list(range(2)))
expected = np.array(val[::-2, :][:, ::-1],
order=order_to,
dtype=dtype_to)
result = pstdata[::-2, ::-1].read(
order=order_to,
dtype=dtype_to,
force_python_only=force_python_only)
assert result.val.dtype == dtype_to
assert (order_to == 'F'
and result.val.flags['F_CONTIGUOUS']
) or (order_to == 'C' and
result.val.flags['C_CONTIGUOUS'])
assert np.array_equal(result.val, expected)
def cmktest_inputs3(self):
from pysnptools.pstreader import PstData
np.random.seed(0)
row_property=None
col_property=None
pstdata = PstData(row=[[1.0,2.0],[3,4],[6,7]],
col=np.array([]),
val = [[],[],[]],
row_property=row_property,
col_property=col_property,
name="test_read")
assert pstdata.row_to_index([[3,4]])[0] == 1
assert np.array_equal(pstdata[1:,:2].row_property,pstdata.row_property[1:])
assert np.array_equal(pstdata[1:,:2].col_property,pstdata.col_property[:2])
logging.info("done with test")
def __init__(self, iid, sid, val, pos=None, name=None, parent_string=None, copyinputs_function=None):
#We don't have a 'super(SnpData, self).__init__()' here because SnpData takes full responsiblity for initializing both its superclasses
self.val = None
if parent_string is not None:
warnings.warn("'parent_string' is deprecated. Use 'name'", DeprecationWarning)
self._row = PstData._fixup_input(iid,empty_creator=lambda ignore:np.empty([0,2],dtype='S'),dtype='S')
self._col = PstData._fixup_input(sid,empty_creator=lambda ignore:np.empty([0],dtype='S'),dtype='S')
self._row_property = PstData._fixup_input(None,count=len(self._row),empty_creator=lambda count:np.empty([count,0],dtype='S'),dtype='S')
self._col_property = PstData._fixup_input(pos,count=len(self._col),empty_creator=lambda count:np.array([[np.nan, np.nan, np.nan]]*count))
self.val = PstData._fixup_input_val(val,row_count=len(self._row),col_count=len(self._col),empty_creator=lambda row_count,col_count:np.empty([row_count,col_count],dtype=np.float64))
self._assert_iid_sid_pos()
self._name = name or parent_string or ""
self._std_string_list = []
def test_writes(self):
#===================================
# Defining sub functions
#===================================
def _oned_int(c):
return list(range(c))
def _oned_str(c):
return [str(i) for i in range(c)]
def _twooned_int(c):
return [[i] for i in range(c)]
def _twooned_str(c):
return [[str(i)] for i in range(c)]
def _twotwod_int(c):
return [[i,i] for i in range(c)]
def _twotwod_str(c):
return [[str(i),"hello"] for i in range(c)]
def _none(c):
return None
def _zero(c):
return np.empty([c,0])
#===================================
# Staring main function
#===================================
logging.info("starting 'test_writes'")
np.random.seed(0)
output_template = "tempdir/pstreader/writes.{0}.{1}"
create_directory_if_necessary(output_template.format(0,"npz"))
i = 0
for row_count in [5,2,1,0]:
for col_count in [4,2,1,0]:
val = np.random.normal(.5,2,size=(row_count,col_count))
for row_or_col_gen in [_oned_int,_oned_str,_twooned_int,_twooned_str,_twotwod_int,_twotwod_str]:
row = row_or_col_gen(row_count)
col = row_or_col_gen(col_count)
for prop_gen in [_oned_int,_oned_str,_twooned_int,_twooned_str,_twotwod_int,_twotwod_str,_none,_zero]:
row_prop = prop_gen(row_count)
col_prop = prop_gen(col_count)
pstdata = PstData(row,col,val,row_prop,col_prop,str(i))
for the_class,suffix in [(PstNpz,"npz"),(PstHdf5,"hdf5")]:
filename = output_template.format(i,suffix)
logging.info(filename)
i += 1
the_class.write(filename,pstdata)
for subsetter in [None, sp.s_[::2,::3]]:
reader = the_class(filename)
_fortesting_JustCheckExists().input(reader)
subreader = reader if subsetter is None else reader[subsetter[0],subsetter[1]]
readdata = subreader.read(order='C')
expected = pstdata if subsetter is None else pstdata[subsetter[0],subsetter[1]].read()
assert np.array_equal(readdata.val,expected.val)
assert np.array_equal(readdata.row,expected.row)
assert np.array_equal(readdata.col,expected.col)
assert np.array_equal(readdata.row_property,expected.row_property)
assert np.array_equal(readdata.col_property,expected.col_property)
try:
os.remove(filename)
except:
pass
logging.info("done with 'test_writes'")
def __init__(self,
iid,
sid,
val,
pos=None,
name=None,
parent_string=None,
copyinputs_function=None,
xp=None,
_require_float32_64=True):
#We don't have a 'super(SnpData, self).__init__()' here because SnpData takes full responsibility for initializing both its superclasses
xp = pstutil.array_module(xp)
self._val = None
if parent_string is not None:
warnings.warn("'parent_string' is deprecated. Use 'name'",
DeprecationWarning)
self._row = PstData._fixup_input(
iid,
empty_creator=lambda ignore: np.empty([0, 2], dtype='str'),
dtype='str')
self._col = PstData._fixup_input(
sid,
empty_creator=lambda ignore: np.empty([0], dtype='str'),
dtype='str')
self._row_property = PstData._fixup_input(
None,
count=len(self._row),
empty_creator=lambda count: np.empty([count, 0], dtype='str'),
dtype='str')
self._col_property = PstData._fixup_input(
pos,
count=len(self._col),
empty_creator=lambda count: np.full([count, 3], np.nan))
self._val = PstData._fixup_input_val(
val,
row_count=len(self._row),
col_count=len(self._col),
empty_creator=lambda row_count, col_count: np.empty(
[row_count, col_count], dtype=np.float64),
_require_float32_64=_require_float32_64,
xp=xp)
self._assert_iid_sid_pos(check_val=True)
self._name = name or parent_string or ""
self._std_string_list = []
self._xp = xp
def val(self, new_value):
self._val = PstData._fixup_input_val(
new_value,
row_count=len(self._row),
col_count=len(self._col),
empty_creator=lambda row_count, col_count: np.empty(
[row_count, col_count], dtype=np.float64))
self._assert_iid0_iid1(check_val=True)
def cmktest_inputs2(self):
from pysnptools.pstreader import PstData
np.random.seed(0)
row_property=None
col_property=None
pstdata = PstData(row=np.array([1.0,3,6]),
col=np.array(["Aa","Bb"]),
val = np.random.normal(.5,2,size=(3,2)),
row_property=row_property,
col_property=col_property,
name="test_read")
assert pstdata.row_to_index([3])[0] == 1
assert pstdata.col_to_index(["Aa"])[0] == 0
assert np.array_equal(pstdata[1:,:2].row_property,pstdata.row_property[1:])
assert np.array_equal(pstdata[1:,:2].col_property,pstdata.col_property[:2])
logging.info("done with test")
def test_inputs2(self):
from pysnptools.pstreader import PstData
np.random.seed(0)
row_property=None
col_property=None
pstdata = PstData(row=np.array([1.0,3,6]),
col=np.array(["Aa","Bb"]),
val = np.random.normal(.5,2,size=(3,2)),
row_property=row_property,
col_property=col_property,
name="test_read")
assert pstdata.row_to_index([3])[0] == 1
assert pstdata.col_to_index(["Aa"])[0] == 0
assert np.array_equal(pstdata[1:,:2].row_property,pstdata.row_property[1:])
assert np.array_equal(pstdata[1:,:2].col_property,pstdata.col_property[:2])
logging.info("done with test")
def test_read(self):
np.random.seed(0)
row_property=np.array([[1.0,2,2.5],[3,4,4.5],[5,6,6.5]])
col_property=np.array([[1.0,2,2.5,1],[3,4,4.5,3]])
pstdata = PstData(row=np.array([[1.0,2],[3,4],[5,6]]),
col=np.array([["A","a"],["B","b"]]),
val = np.random.normal(.5,2,size=(3,2)),
row_property=row_property,
col_property=col_property,
name="test_read")
assert pstdata.row_to_index([np.array([3.0,4])])[0] == 1
assert pstdata.col_to_index([np.array(["A","a"])])[0] == 0
assert np.array_equal(pstdata[1:,:2].row_property,row_property[1:])
assert np.array_equal(pstdata[1:,:2].col_property,col_property[:2])
pstdata2 = pstdata[:2,:2].read()
from pysnptools.kernelreader.test import _fortesting_JustCheckExists
_fortesting_JustCheckExists().input(pstdata)
_fortesting_JustCheckExists().input(pstdata2)
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata3 = pstdata[[],:].read()
assert pstdata3.val.shape[0] == 0 and pstdata3.val.shape[1]==2
pstdata.val = pstdata.val.copy(order='F')
pstdata2 = pstdata[:2,:2].read()
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata2 = pstdata[:2,:2].read(order='F')
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata2 = pstdata[:2,:2].read(order='A')
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype=None,order='C')
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype='float32',order='C')
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2].astype(dtype='float32'), decimal=10)
pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype='float32',order=None)
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2].astype(dtype='float32'), decimal=10)
pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype=None,order='F')
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata4 = pstdata[::,::].read(force_python_only=True)
np.testing.assert_array_almost_equal(pstdata4.val, pstdata.val, decimal=10)
logging.info("done with test")
def test_read(self):
np.random.seed(0)
row_property=np.array([[1.0,2,2.5],[3,4,4.5],[5,6,6.5]])
col_property=np.array([[1.0,2,2.5,1],[3,4,4.5,3]])
pstdata = PstData(row=np.array([[1.0,2],[3,4],[5,6]]),
col=np.array([["A","a"],["B","b"]]),
val = np.random.normal(.5,2,size=(3,2)),
row_property=row_property,
col_property=col_property,
name="test_read")
assert pstdata.row_to_index([np.array([3.0,4])])[0] == 1
assert pstdata.col_to_index([np.array(["A","a"])])[0] == 0
assert np.array_equal(pstdata[1:,:2].row_property,row_property[1:])
assert np.array_equal(pstdata[1:,:2].col_property,col_property[:2])
pstdata2 = pstdata[:2,:2].read()
from pysnptools.kernelreader.test import _fortesting_JustCheckExists
_fortesting_JustCheckExists().input(pstdata)
_fortesting_JustCheckExists().input(pstdata2)
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata3 = pstdata[[],:].read()
assert pstdata3.val.shape[0] == 0 and pstdata3.val.shape[1]==2
pstdata.val = pstdata.val.copy(order='F')
pstdata2 = pstdata[:2,:2].read()
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata2 = pstdata[:2,:2].read(order='F')
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata2 = pstdata[:2,:2].read(order='A')
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype=None,order='C')
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype='float32',order='C')
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2].astype(dtype='float32'), decimal=10)
pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype='float32',order=None)
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2].astype(dtype='float32'), decimal=10)
pstdata2 = pstdata[:2,:2].read(force_python_only=True,dtype=None,order='F')
np.testing.assert_array_almost_equal(pstdata2.val, pstdata.val[:2,:2], decimal=10)
pstdata4 = pstdata[::,::].read(force_python_only=True)
np.testing.assert_array_almost_equal(pstdata4.val, pstdata.val, decimal=10)
logging.info("done with test")
def __init__(self, filename, count_A1=None, iid=None, sid=None, pos=None, skip_format_check=False): #!!!document these new optionals. they are here
super(Bed, self).__init__()
self._ran_once = False
self._file_pointer = None
self.filename = filename
if count_A1 is None:
warnings.warn("'count_A1' was not set. For now it will default to 'False', but in the future it will default to 'True'", FutureWarning)
count_A1 = False
self.count_A1 =count_A1
self.skip_format_check = skip_format_check
if iid is not None:
self._row = PstData._fixup_input(iid,empty_creator=lambda ignore:np.empty([0,2],dtype='S'),dtype='S')
if sid is not None:
self._col = PstData._fixup_input(sid,empty_creator=lambda ignore:np.empty([0],dtype='S'),dtype='S')
if pos is not None:
self._col_property = PstData._fixup_input(pos,count=len(self._col),empty_creator=lambda count:np.array([[np.nan, np.nan, np.nan]]*count))
def val(self, new_value):
self._val = PstData._fixup_input_val(
new_value,
row_count=len(self._row),
col_count=len(self._col),
empty_creator=lambda row_count, col_count: np.empty(
[row_count, col_count, 3], dtype=np.float64)
) #!!!Replace empty with my FillNA method?
self._assert_iid_sid_pos(check_val=True)
def test_big_npz(self):
logging.info("in test_big_npz")
n = 1000
pstdata = PstData(row=range(n-1),col=range(n+1),val=np.zeros([n-1,n+1]))
output = "tempdir/pstreader/big.npz"
create_directory_if_necessary(output)
PstNpz.write(output,pstdata)
pstnpz = PstNpz(output)
pstdata1 = pstnpz[::2,::4].read()
pstdata2 = pstnpz.read(order='A')
assert pstdata2.val.flags['C_CONTIGUOUS']
pstdata = PstData(row=range(n-1),col=range(n+1),val=np.zeros([n-1,n+1],order='F'))
PstNpz.write(output,pstdata)
pstnpz = PstNpz(output)
pstdata2 = pstnpz.read(order='A')
pstdata2.val.flags['F_CONTIGUOUS']
print("done")
def test_inputs4(self):
from pysnptools.pstreader import PstData
pstdata = PstData(row=None,
col=None,
val = None,
row_property=None,
col_property=None,
name="test_read")
assert pstdata.row_count == 0 and pstdata.col_count == 0 and pstdata.val.shape[0] == 0 and pstdata.val.shape[1]==0 and len(pstdata.row_property)==0 and len(pstdata.col_property)==0
logging.info("done with test")
def _empty_inner(self, row, col, filename, row_property, col_property, order, dtype, val_shape):
self._ran_once = True
self._dtype = np.dtype(dtype)
self._order = order
row = PstData._fixup_input(row)
col = PstData._fixup_input(col)
row_property = PstData._fixup_input(row_property,count=len(row))
col_property = PstData._fixup_input(col_property,count=len(col))
with open(filename,'wb') as fp:
np.save(fp, np.array([_magic_number]))
np.save(fp, np.array(["pstmemmap"])) #name of file format
np.save(fp, np.array([2])) #file format version
np.save(fp, row)
np.save(fp, col)
np.save(fp, row_property)
np.save(fp, col_property)
np.save(fp, np.array([self._dtype]))
np.save(fp, np.array([self._order]))
np.save(fp, np.array([val_shape]))
self._offset = fp.tell()
logging.info("About to start allocating memmap '{0}'".format(filename))
shape = (len(row),len(col)) if val_shape is None else (len(row),len(col),val_shape)
val = np.memmap(filename, offset=self._offset, dtype=dtype, mode="r+", order=order, shape=shape)
logging.info("Finished allocating memmap '{0}'. Size is {1:,}".format(filename,os.path.getsize(filename)))
PstData.__init__(self,row,col,val,row_property,col_property,name="np.memmap('{0}')".format(filename))
def __init__(self,
iid=None,
iid0=None,
iid1=None,
val=None,
name=None,
parent_string=None,
xp=None
): #!!!autodoc doesn't generate good doc for this constructor
#We don't have a 'super(KernelData, self).__init__()' here because KernelData takes full responsibility for initializing both its superclasses
xp = pstutil.array_module(xp)
self._val = None
#!!why does SnpData __init__ have a copy_inputs, but KernelData doesn't?
assert (iid is None) != (
iid0 is None and iid1 is None
), "Either 'iid' or both 'iid0' 'iid1' must be provided."
assert name is None or parent_string is None, "Can't set both 'name' and the deprecated 'parent_string'"
if parent_string is not None:
warnings.warn("'parent_string' is deprecated. Use 'name'",
DeprecationWarning)
if iid is not None:
self._row = PstData._fixup_input(
iid,
empty_creator=lambda ignore: np.empty([0, 2], dtype='str'),
dtype='str')
self._col = self._row
else:
self._row = PstData._fixup_input(
iid0,
empty_creator=lambda ignore: np.empty([0, 2], dtype='str'),
dtype='str')
self._col = PstData._fixup_input(
iid1,
empty_creator=lambda ignore: np.empty([0, 2], dtype='str'),
dtype='str')
self._row_property = PstData._fixup_input(
None,
count=len(self._row),
empty_creator=lambda count: np.empty([count, 0], dtype='str'),
dtype='str')
self._col_property = PstData._fixup_input(
None,
count=len(self._col),
empty_creator=lambda count: np.empty([count, 0], dtype='str'),
dtype='str')
self._val = PstData._fixup_input_val(
val,
row_count=len(self._row),
col_count=len(self._col),
empty_creator=lambda row_count, col_count: xp.empty(
[row_count, col_count], dtype=xp.float64),
xp=xp)
self._assert_iid0_iid1(check_val=True)
self._name = name or parent_string or ""
self._std_string_list = []
self._xp = xp
def test2(self):
old_dir = os.getcwd()
os.chdir(os.path.dirname(os.path.realpath(__file__)))
filename = "tempdir/x.pst.memmap"
pstutil.create_directory_if_necessary(filename)
a = PstMemMap.empty(row=['a','b','c'],col=['y','z'],filename=filename,row_property=['A','B','C'],order="F",dtype=np.float64)
b = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
pointer1, read_only_flag = a.val.__array_interface__['data']
a.val+=1
a.val+=b.val
pointer2, read_only_flag = a.val.__array_interface__['data']
assert pointer1==pointer2
os.chdir(old_dir)
def allclose(self, value, equal_nan=True):
'''
:param value: Other object with which to compare.
:type value: :class:`SnpData`
:param equal_nan: (Default: True) Tells if NaN in :attr:`SnpData.val` should be treated as regular values when testing equality.
:type equal_nan: bool
>>> import numpy as np
>>> snpdata5 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]], pos=[[0,0,0],[0,0,0],[0,0,0]])
>>> snpdata6 = SnpData(iid=[['fam0','iid0'],['fam0','iid1']], sid=['snp334','snp349','snp921'], val=[[0.,2.,0.],[0.,1.,np.nan]], pos=[[0,0,0],[0,0,0],[0,0,0]])
>>> print(snpdata5.allclose(snpdata6)) #True, if we consider the NaN as regular values, all the arrays have the same values.
True
>>> print(snpdata5.allclose(snpdata6,equal_nan=False)) #False, if we consider the NaN as special values, all the arrays are not equal.
False
'''
return PstData.allclose(self, value, equal_nan=equal_nan)
def _read_pstdata(self):
bim_list = []
val_list_list = []
with open(self.filename, "r") as fp:
header = fp.readline()
iid_string_list = header.strip().split()[1:]
iid = np.array([
self.extract_iid_function(iid_string)
for iid_string in iid_string_list
],
dtype='str')
val_list = []
zerofloat = float('0'[0])
missing_char = "?"[0]
for line_index, line in enumerate(fp):
if line_index % 1000 == 0:
logging.info(
"reading sid and iid info from line {0} of file '{1}'".
format(line_index, self.filename))
sid_string_rest = line.strip().split()
sid_string = sid_string_rest[0]
rest = [] if len(sid_string_rest) == 1 else sid_string_rest[1]
assert len(rest) == len(iid)
bim_list.append(self.extract_sid_pos_function(sid_string))
val_list = np.array([
float(val) - zerofloat if val != missing_char else np.NaN
for val in rest
])
val_list_list.append(val_list)
col = np.array([bim[1] for bim in bim_list], dtype='str')
col_property = np.array([[bim[0], bim[2], bim[3]] for bim in bim_list],
dtype=np.float64)
val = np.zeros((len(iid), len(col)))
for col_index in range(len(col)):
val[:, col_index] = val_list_list[col_index]
return PstData(iid,
col,
val,
col_property=col_property,
name=self.filename)
def test_subset(self):
np.random.seed(0)
row_property = np.array([[1.0, 2, 2.5], [3, 4, 4.5], [5, 6, 6.5]])
col_property = np.array([[1.0, 2, 2.5, 1], [3, 4, 4.5, 3]])
val = np.random.normal(.5, 2, size=(3, 2))
pstdata = PstData(row=np.array([[1.0, 2], [3, 4], [5, 6]]),
col=np.array([["A", "a"], ["B", "b"]]),
val=val,
row_property=row_property,
col_property=col_property,
name="test_read")
assert np.array_equal(pstdata[-1:0:-1, :].read().val, val[-1:0:-1, :])
assert pstdata[-1, -1].read().val[0, 0] == val[-1, -1]
assert np.array_equal(pstdata[[-1, 0], [-1, 0]].read().val,
val[[-1, 0], :][:, [-1, 0]])
assert np.array_equal(
pstdata[[True, False, True], [False, True]].read().val,
val[[True, False, True], [False, True]].reshape(2, 1))
assert pstdata[0, 0].read().val[0, 0] == val[0, 0]
assert np.array_equal(pstdata[1::2, 1::2].read().val, val[1::2, 1::2])
logging.info("done with test")
def __init__(self, iid=None, iid0=None, iid1=None, val=None, name=None, parent_string=None): #!!!autodoc doesn't generate good doc for this constructor
#!!why does SnpData __init__ have a copy_inputs, but KernelData doesn't?
assert (iid is None) != (iid0 is None and iid1 is None), "Either 'iid' or both 'iid0' 'iid1' must be provided."
assert name is None or parent_string is None, "Can't set both 'name' and the deprecated 'parent_string'"
if parent_string is not None:
warnings.warn("'parent_string' is deprecated. Use 'name'", DeprecationWarning)
if iid is not None:
self._row = PstData._fixup_input(iid,empty_creator=lambda ignore:np.empty([0,2],dtype=str))
self._col = self._row
else:
self._row = PstData._fixup_input(iid0,empty_creator=lambda ignore:np.empty([0,2],dtype=str))
self._col = PstData._fixup_input(iid1,empty_creator=lambda ignore:np.empty([0,2],dtype=str))
self._row_property = PstData._fixup_input(None,count=len(self._row),empty_creator=lambda count:np.empty([count,0],dtype=str))
self._col_property = PstData._fixup_input(None,count=len(self._col),empty_creator=lambda count:np.empty([count,0],dtype=str))
self.val = PstData._fixup_input_val(val,row_count=len(self._row),col_count=len(self._col),empty_creator=lambda row_count,col_count:np.empty([row_count,col_count],dtype=np.float64))
self._assert_iid0_iid1()
self._name = name or parent_string or ""
self._std_string_list = []
fn = '../examples/tiny.pst.memmap'
os.getcwd()
print((os.path.exists(fn)))
pst_mem_map = PstMemMap(fn)
print((pst_mem_map.val[0,1]))
if False:
a=np.ndarray([2,3])
pointer, read_only_flag = a.__array_interface__['data']
print(pointer)
a*=2
pointer, read_only_flag = a.__array_interface__['data']
print(pointer)
a = PstMemMap.empty(row=['a','b','c'],col=['y','z'],filename=r'c:\deldir\a.memmap',row_property=['A','B','C'],order="F",dtype=np.float64)
b = PstData(row=['a','b','c'],col=['y','z'],val=[[1,2],[3,4],[np.nan,6]],row_property=['A','B','C'])
pointer, read_only_flag = a.val.__array_interface__['data']
print(pointer)
a.val+=1
a.val+=b.val
pointer, read_only_flag = a.val.__array_interface__['data']
print(pointer)
suites = getTestSuite()
r = unittest.TextTestRunner(failfast=True)
ret = r.run(suites)
assert ret.wasSuccessful()
result = doctest.testmod(optionflags=doctest.ELLIPSIS)
assert result.failed == 0, "failed doc test: " + __file__
def read(self,
order='F',
dtype=np.float64,
force_python_only=False,
view_ok=False,
num_threads=None):
"""Reads the matrix values and returns a :class:`.PstData` (with :attr:`PstData.val` property containing a new ndarray of the matrix values).
:param order: {'F' (default), 'C', 'A'}, optional -- Specify the order of the ndarray. If order is 'F' (default),
then the array will be in F-contiguous order (row-index varies the fastest).
If order is 'C', then the returned array will be in C-contiguous order (col-index varies the fastest).
If order is 'A', then the :attr:`PstData.val`
ndarray may be in any order (either C-, Fortran-contiguous).
:type order: string or None
:param dtype: {numpy.float64 (default), numpy.float32}, optional -- The data-type for the :attr:`PstData.val` ndarray.
:type dtype: data-type
:param force_python_only: optional -- If False (default), may use outside library code. If True, requests that the read
be done without outside library code.
:type force_python_only: bool
:param view_ok: optional -- If False (default), allocates new memory for the :attr:`PstData.val`'s ndarray. If True,
if practical and reading from a :class:`PstData`, will return a new
:class:`PstData` with a ndarray shares memory with the original :class:`PstData`.
Typically, you'll also wish to use "order='A'" to increase the chance that sharing will be possible.
Use these parameters with care because any change to either ndarray will effect
the others. Also keep in mind that :meth:`read` relies on ndarray's mechanisms to decide whether to actually
share memory and so it may ignore your suggestion and allocate a new ndarray anyway.
:type view_ok: bool
:param num_threads: optional -- The number of threads with which to read data. Defaults to all available
processors. Can also be set with these environment variables (listed in priority order):
'PST_NUM_THREADS', 'NUM_THREADS', 'MKL_NUM_THREADS'.
:type num_threads: None or int
:rtype: :class:`.PstData`
Calling the method again causes the matrix values to be re-read and creates a new in-memory :class:`.PstData` with a new ndarray of matrix values.
If you request the values for only a subset of the sids or iids, (to the degree practical) only that subset will be read from disk.
:Example:
>>> from pysnptools.pstreader import PstHdf5
>>> from pysnptools.util import example_file # Download and return local file name
>>> hdf5_file = example_file('pysnptools/examples/toydata.iidmajor.snp.hdf5')
>>> on_disk = PstHdf5(hdf5_file) # Specify matrix data on disk
>>> pstdata1 = on_disk.read() # Read all the matrix data returning a PstData instance
>>> print(type(pstdata1.val).__name__) # The PstData instance contains a ndarray of the data.
ndarray
>>> subset_pstdata = on_disk[:,::2].read() # From the disk, read matrix values for every other sid
>>> print(subset_pstdata.val[0,0]) # Print the first matrix value in the subset
1.0
>>> subsub_pstdata = subset_pstdata[:10,:].read(order='A',view_ok=True) # Create an in-memory subset of the subset with matrix values for the first ten iids. Share memory if practical.
>>> import numpy as np
>>> # print(np.may_share_memory(subset_snpdata.val, subsub_snpdata.val)) # Do the two ndarray's share memory? They could. Currently they won't.
"""
dtype = np.dtype(dtype)
val = self._read(None, None, order, dtype, force_python_only, view_ok,
num_threads)
from pysnptools.pstreader import PstData
ret = PstData(self.row,
self.col,
val,
row_property=self.row_property,
col_property=self.col_property,
name=str(self))
return ret
def test_writes(self):
#===================================
# Defining sub functions
#===================================
def _oned_int(c):
return range(c)
def _oned_str(c):
return [str(i).encode('ascii') for i in range(c)]
def _twooned_int(c):
return [[i] for i in range(c)]
def _twooned_str(c):
return [[str(i).encode('ascii')] for i in range(c)]
def _twotwod_int(c):
return [[i, i] for i in range(c)]
def _twotwod_str(c):
return [[str(i).encode('ascii'), b"hello"] for i in range(c)]
#def _twotwod_U(c):
# return [[str(i).encode('UTF-8'),u"hello"] for i in range(c)]
def _none(c):
return None
def _zero(c):
return np.empty([c, 0], dtype='S')
#===================================
# Starting main function
#===================================
logging.info("starting 'test_writes'")
np.random.seed(0)
temp_dir = tempfile.TemporaryDirectory("pstreader")
output_template = temp_dir.name + '/writes.{0}.{1}'
i = 0
for row_count in [5, 2, 1, 0]:
for col_count in [4, 2, 1, 0]:
for val_shape in [3, None, 1]:
val = np.random.normal(.5, 2, size=(
row_count,
col_count)) if val_shape is None else np.random.normal(
.5, 2, size=(row_count, col_count, val_shape))
for row_or_col_gen in [
_oned_int, _oned_str, _twooned_int, _twooned_str,
_twotwod_int, _twotwod_str
]: #!!!,_twotwod_U can't roundtrop Unicode in hdf5
row = row_or_col_gen(row_count)
col = row_or_col_gen(col_count)
for prop_gen in [
_none, _oned_str, _oned_int, _twooned_int,
_twooned_str, _twotwod_int, _twotwod_str, _zero
]: #!!!_twotwod_U can't round trip Unicode because Hdf5 doesn't like it.
row_prop = prop_gen(row_count)
col_prop = prop_gen(col_count)
pstdata = PstData(row, col, val, row_prop,
col_prop, str(i))
for the_class, suffix in [(PstMemMap, "memmap"),
(PstHdf5, "hdf5"),
(PstNpz, "npz")]:
filename = output_template.format(i, suffix)
logging.info(filename)
i += 1
the_class.write(filename, pstdata)
reader = the_class(
filename
) if suffix != 'hdf5' else the_class(
filename, block_size=3)
_fortesting_JustCheckExists().input(reader)
for subsetter in [None, np.s_[::2, ::3]]:
subreader = reader if subsetter is None else reader[
subsetter[0], subsetter[1]]
expected = pstdata if subsetter is None else pstdata[
subsetter[0], subsetter[1]].read()
for order in ['C', 'F', 'A']:
for force_python_only in [True, False]:
readdata = subreader.read(
order=order,
force_python_only=
force_python_only)
assert np.array_equal(
readdata.val, expected.val)
assert np.array_equal(
readdata.row, expected.row)
assert np.array_equal(
readdata.col, expected.col)
assert np.array_equal(
readdata.row_property,
expected.row_property
) or (
readdata.row_property.shape[1]
== 0 and
expected.row_property.shape[1]
== 0)
assert np.array_equal(
readdata.col_property,
expected.col_property
) or (
readdata.col_property.shape[1]
== 0 and
expected.col_property.shape[1]
== 0)
if suffix in {'memmap', 'hdf5'}:
reader.flush()
os.remove(filename)
temp_dir.cleanup()
logging.info("done with 'test_writes'")