def sub_test_store(self, readWrite):
td = TrainData()
x,y,w = self.createSimpleArray('int32'), self.createSimpleArray('float32'), self.createSimpleArray('int32')
x_orig=x.copy()
x2,y2,_ = self.createSimpleArray('float32'), self.createSimpleArray('float32'), self.createSimpleArray('int32')
x2_orig=x2.copy()
y_orig=y.copy()
td._store([x,x2], [y,y2], [w])
if readWrite:
td.writeToFile("testfile.tdjctd")
td = TrainData()
td.readFromFile("testfile.tdjctd")
os.system('rm -f testfile.tdjctd')
shapes = td.getNumpyFeatureShapes()
self.assertEqual([[3, 5, 6], [1], [3, 5, 6], [1]], shapes,"shapes")
self.assertEqual(2, td.nFeatureArrays())
self.assertEqual(2, td.nTruthArrays())
self.assertEqual(1, td.nWeightArrays())
f = td.transferFeatureListToNumpy(False)
t = td.transferTruthListToNumpy(False)
w = td.transferWeightListToNumpy(False)
xnew = SimpleArray(f[0],np.array(f[1],dtype='int64'))
self.assertEqual(x_orig, xnew)
xnew = SimpleArray(f[2],np.array(f[3],dtype='int64'))
self.assertEqual(x2_orig, xnew)
ynew = SimpleArray(t[0],np.array(t[1],dtype='int64'))
self.assertEqual(y_orig, ynew)
def test_transferToNumpyInt(self):
print('TestSimpleArray: transferToNumpyInt')
arr, rs = self.createNumpy('int32')
a = SimpleArray(arr, rs)
narr, nrs = a.transferToNumpy()
diff = np.max(np.abs(narr - arr))
diff += np.max(np.abs(nrs - rs))
self.assertTrue(diff < 0.000001)
def test_split(self):
print('TestSimpleArray: split')
arr, rs = self.createNumpy('float32')
a = SimpleArray(arr, rs)
arrs, rss = arr[:rs[2]], rs[:3]
b = SimpleArray(arrs, rss)
asplit = a.split(2)
self.assertEqual(asplit, b)
def test_SimpleArrayRead(self):
print('TestCompatibility SimpleArray')
a = SimpleArray()
a.readFromFile("simpleArray_previous.djcsa")
arr = np.load("np_arr.npy")
rs = np.load("np_rs.npy")
b = SimpleArray(arr, rs)
self.assertEqual(a, b)
def test_createFromNumpyInt(self):
print('TestSimpleArray: createFromNumpyInt')
arr, rs = self.createNumpy('int32')
a = SimpleArray(dtype='int32')
a.createFromNumpy(arr, rs)
narr, nrs = a.copyToNumpy()
diff = np.max(np.abs(narr - arr))
diff += np.max(np.abs(nrs - rs))
self.assertTrue(diff < 0.000001)
def test_SimpleArrayRead(self):
print('TestCompatibility SimpleArray')
a = SimpleArray()
a.readFromFile("simpleArray_previous.djcsa")
arr = np.load("np_arr.npy")
#FIXME: this array was actually wrong
arr = arr[:100]
rs = np.load("np_rs.npy")
b = SimpleArray(arr, rs)
self.assertEqual(a, b)
def test_dynamicTypeChange(self):
print('TestSimpleArray: dynamicTypeChange')
arr, rs = self.createNumpy('int32')
name = "lala"
a = SimpleArray(dtype='float32', name=name)
fnames = ["a", "b", "c", "d", "e", "f"]
a.setFeatureNames(fnames)
a.createFromNumpy(arr, rs)
self.assertTrue(a.featureNames() == fnames)
self.assertTrue(a.name() == name)
def test_TrainDataRead(self):
print('TestCompatibility TrainData')
td = TrainData()
td.readFromFile('trainData_previous.djctd')
self.assertEqual(td.nFeatureArrays(), 1)
arr = np.load("np_arr.npy")
rs = np.load("np_rs.npy")
b = SimpleArray(arr, rs)
a = td.transferFeatureListToNumpy(False)
a, rs = a[0], a[1]
a = SimpleArray(a, np.array(rs, dtype='int64'))
self.assertEqual(a, b)
def test_split(self):
print('TestTrainData: split')
a = self.createSimpleArray('int32')
b = self.createSimpleArray('float32',600)
c = self.createSimpleArray('int32')
d = self.createSimpleArray('float32',400)
all_orig = [a.copy(),b.copy(),c.copy(),d.copy()]
all_splitorig = [sa.split(2) for sa in all_orig]
td = TrainData()
td._store([a,b], [c,d], [])
tdb = td.split(2)
f = tdb.transferFeatureListToNumpy(False)
t = tdb.transferTruthListToNumpy(False)
_ = tdb.transferWeightListToNumpy(False)
all_split = [SimpleArray(f[0],f[1]), SimpleArray(f[2],f[3]),
SimpleArray(t[0],t[1]), SimpleArray(t[2],t[3])]
self.assertEqual(all_splitorig,all_split)
def test_slice(self):
print('TestTrainData: skim')
a = self.createSimpleArray('int32', 600)
b = self.createSimpleArray('float32', 600)
d = self.createSimpleArray('float32', 600)
a_slice = a.getSlice(2, 3)
b_slice = b.getSlice(2, 3)
d_slice = d.getSlice(2, 3)
td = TrainData()
td._store([a, b], [d], [])
td_slice = td.getSlice(2, 3)
fl = td_slice.transferFeatureListToNumpy(False)
tl = td_slice.transferTruthListToNumpy(False)
a_tdslice = SimpleArray(fl[0], fl[1])
b_tdslice = SimpleArray(fl[2], fl[3])
d_tdslice = SimpleArray(tl[0], tl[1])
self.assertEqual(a_slice, a_tdslice)
self.assertEqual(b_slice, b_tdslice)
self.assertEqual(d_slice, d_tdslice)
#test skim
td.skim(2)
fl = td.transferFeatureListToNumpy(False)
tl = td.transferTruthListToNumpy(False)
a_tdslice = SimpleArray(fl[0], fl[1])
b_tdslice = SimpleArray(fl[2], fl[3])
d_tdslice = SimpleArray(tl[0], tl[1])
self.assertEqual(a_slice, a_tdslice)
self.assertEqual(b_slice, b_tdslice)
self.assertEqual(d_slice, d_tdslice)
def _convertToCppType(self,a,helptext):
saout=None
if str(type(a)) == "<class 'DeepJetCore.SimpleArray.SimpleArray'>":
saout = a.sa
elif str(type(a)) == "<type 'numpy.ndarray'>" or str(type(a)) == "<class 'numpy.ndarray'>":
rs = np.array([])
a = SimpleArray(a,rs)
saout = a.sa
else:
raise ValueError("TrainData._convertToCppType MUST produce either a list of numpy arrays or a list of DeepJetCore simpleArrays!")
if saout.hasNanOrInf():
raise ValueError("TrainData._convertToCppType: the "+helptext+" array "+saout.name()+" has NaN or inf entries")
return saout
def test_append(self):
print('TestSimpleArray: append')
arr, rs = self.createNumpy('float32')
arr2, _ = self.createNumpy('float32')
a = SimpleArray(arr, rs)
aa = SimpleArray(arr2, rs)
a.append(aa)
arr2 = np.concatenate([arr, arr2], axis=0)
rs2 = rs.copy()[1:]
rs2 += rs[-1]
rs2 = np.concatenate([rs, rs2], axis=0)
b = SimpleArray(arr2, rs2)
self.assertEqual(a, b)
def test_equal(self):
print('TestSimpleArray: equal')
arr, rs = self.createNumpy('float32')
a = SimpleArray()
a.createFromNumpy(arr, rs)
b = SimpleArray()
b.createFromNumpy(arr, rs)
self.assertEqual(a, b)
b = a.copy()
self.assertEqual(a, b)
b.setFeatureNames(["a", "b", "c", "d", "e", "f"])
self.assertNotEqual(a, b)
def test_writeRead(self):
print('TestSimpleArray: writeRead')
arr, rs = self.createNumpy('float32')
a = SimpleArray(arr, rs)
a.setName("myname")
a.setFeatureNames(["a", "b", "c", "d", "e", "f"])
a.writeToFile("testfile.djcsa")
b = SimpleArray()
b.readFromFile("testfile.djcsa")
os.system('rm -f testfile.djcsa')
#os.system("rf -f testfile")
ad, ars = a.copyToNumpy()
bd, brs = b.copyToNumpy()
diff = np.max(np.abs(ad - bd))
diff += np.max(np.abs(ars - brs))
self.assertTrue(diff == 0)
def test_name(self):
print('TestSimpleArray: name')
arr, rs = self.createNumpy('float32')
a = SimpleArray(arr, rs)
a.setName("myname")
self.assertEqual("myname", a.name())
def createSimpleArray(self, dtype, length=500, shape=None):
arr = np.array(np.random.rand(length, 3, 5, 6) * 100., dtype=dtype)
rs = np.array([0, 100, 230, length], dtype='int64')
return SimpleArray(arr, rs)