def writeOutPrediction(self, predicted, features, truth, weights,
outfilename, inputfile):
# predicted will be a list
print('writeout')
print('predicted', predicted[0].shape)
print('features', features[0].shape)
print('truth', truth[0].shape)
def unroll(a):
a = np.reshape(a,
[a.shape[0], a.shape[1] * a.shape[2], a.shape[3]])
return a
#unroll to event x vector
# first 100 are enough for now
parr = predicted[0][:100, ...] #unroll(predicted[0])
farr = features[0][:100, ...] #unroll(features[0])
tarr = truth[0][:100, ...] #unroll(truth[0])
from DeepJetCore.TrainData import TrainData
#use traindata as data storage
td = TrainData()
td._store([parr, farr, tarr], [], [])
td.writeToFile(outfilename)
return
from root_numpy import array2root
out = np.core.records.fromarrays(
[
parr[:, :, 0],
parr[:, :, 1],
parr[:, :, 2],
parr[:, :, 3],
parr[:, :, 4],
parr[:, :, 5],
parr[:, :, 6],
parr[:, :, 7],
parr[:, :, 9],
parr[:, :, 10],
tarr[:, :, 0],
tarr[:, :, 1],
tarr[:, :, 2],
tarr[:, :, 3],
tarr[:, :, 4],
tarr[:, :, 5],
tarr[:, :, 6],
tarr[:, :, 7],
farr[:, :, 0],
farr[:, :, 1],
farr[:, :, 2],
farr[:, :, 3],
farr[:, :, 4],
],
names=
'p_beta, p_posx, p_posy, p_ID0, p_ID1, p_ID2, p_dim1, p_dim2, p_ccoords1, p_coords2, t_mask, t_posx, t_posy, t_ID0, t_ID1, tID_2, t_dim1, t_dim2, f_r, f_g, f_b, f_x, f_y'
)
array2root(out, outfilename, 'tree')
'''
def test_slice(self):
print('TestTrainData: skim')
a = self.createSimpleArray('int32', 600)
b = self.createSimpleArray('float32', 600)
d = self.createSimpleArray('float32', 600)
a_slice = a.getSlice(2, 3)
b_slice = b.getSlice(2, 3)
d_slice = d.getSlice(2, 3)
td = TrainData()
td._store([a, b], [d], [])
td_slice = td.getSlice(2, 3)
fl = td_slice.transferFeatureListToNumpy(False)
tl = td_slice.transferTruthListToNumpy(False)
a_tdslice = SimpleArray(fl[0], fl[1])
b_tdslice = SimpleArray(fl[2], fl[3])
d_tdslice = SimpleArray(tl[0], tl[1])
self.assertEqual(a_slice, a_tdslice)
self.assertEqual(b_slice, b_tdslice)
self.assertEqual(d_slice, d_tdslice)
#test skim
td.skim(2)
fl = td.transferFeatureListToNumpy(False)
tl = td.transferTruthListToNumpy(False)
a_tdslice = SimpleArray(fl[0], fl[1])
b_tdslice = SimpleArray(fl[2], fl[3])
d_tdslice = SimpleArray(tl[0], tl[1])
self.assertEqual(a_slice, a_tdslice)
self.assertEqual(b_slice, b_tdslice)
self.assertEqual(d_slice, d_tdslice)
def sub_test_store(self, readWrite):
td = TrainData()
x,y,w = self.createSimpleArray('int32'), self.createSimpleArray('float32'), self.createSimpleArray('int32')
x_orig=x.copy()
x2,y2,_ = self.createSimpleArray('float32'), self.createSimpleArray('float32'), self.createSimpleArray('int32')
x2_orig=x2.copy()
y_orig=y.copy()
td._store([x,x2], [y,y2], [w])
if readWrite:
td.writeToFile("testfile.tdjctd")
td = TrainData()
td.readFromFile("testfile.tdjctd")
os.system('rm -f testfile.tdjctd')
shapes = td.getNumpyFeatureShapes()
self.assertEqual([[3, 5, 6], [1], [3, 5, 6], [1]], shapes,"shapes")
self.assertEqual(2, td.nFeatureArrays())
self.assertEqual(2, td.nTruthArrays())
self.assertEqual(1, td.nWeightArrays())
f = td.transferFeatureListToNumpy(False)
t = td.transferTruthListToNumpy(False)
w = td.transferWeightListToNumpy(False)
xnew = SimpleArray(f[0],np.array(f[1],dtype='int64'))
self.assertEqual(x_orig, xnew)
xnew = SimpleArray(f[2],np.array(f[3],dtype='int64'))
self.assertEqual(x2_orig, xnew)
ynew = SimpleArray(t[0],np.array(t[1],dtype='int64'))
self.assertEqual(y_orig, ynew)
def test_KerasDTypes(self):
print('TestTrainData: split')
a = self.createSimpleArray('int32')
b = self.createSimpleArray('float32', 600)
c = self.createSimpleArray('int32')
d = self.createSimpleArray('float32', 400)
td = TrainData()
td._store([a, b], [c, d], [])
#data, rs, data, rs
self.assertEqual(td.getKerasFeatureDTypes(),
['int32', 'int64', 'float32', 'int64'])
def writeOutPrediction(self, predicted, features, truth, weights,
outfilename, inputfile):
# predicted will be a list
print('writeout')
print('predicted', predicted[0].shape)
print('features', features[0].shape)
print('truth', truth[0].shape)
parr = predicted[0] #unroll(predicted[0])
farr = features[0] #unroll(features[0])
tarr = truth[0] #unroll(truth[0])
from DeepJetCore.TrainData import TrainData
#use traindata as data storage
td = TrainData()
td._store([parr, farr, tarr], [], [])
td.writeToFile(outfilename)
def premixfile(i):
eventsperround = 100
neventstotal = nEvents
nPUpremix = nPU
nfilespremix = 5
filearr = pmf(allfiles,
neventstotal,
nPUpremix,
nfilespremix=5,
eventsperround=100)
print('nevents', filearr.shape[0])
td = TrainData()
td._store([filearr], [], [])
print('..writing ' + str(i))
td.writeToFile(outputDir + '/' + str(i) + '_mix.djctd')
del td
def test_split(self):
print('TestTrainData: split')
a = self.createSimpleArray('int32')
b = self.createSimpleArray('float32',600)
c = self.createSimpleArray('int32')
d = self.createSimpleArray('float32',400)
all_orig = [a.copy(),b.copy(),c.copy(),d.copy()]
all_splitorig = [sa.split(2) for sa in all_orig]
td = TrainData()
td._store([a,b], [c,d], [])
tdb = td.split(2)
f = tdb.transferFeatureListToNumpy(False)
t = tdb.transferTruthListToNumpy(False)
_ = tdb.transferWeightListToNumpy(False)
all_split = [SimpleArray(f[0],f[1]), SimpleArray(f[2],f[3]),
SimpleArray(t[0],t[1]), SimpleArray(t[2],t[3])]
self.assertEqual(all_splitorig,all_split)
def test_AddToFile(self):
print('TestTrainData: AddToFile')
td = TrainData()
x, y, w = self.createSimpleArray('int32'), self.createSimpleArray(
'float32'), self.createSimpleArray('int32')
xo, yo, wo = x.copy(), y.copy(), w.copy()
x2, y2, _ = self.createSimpleArray('float32'), self.createSimpleArray(
'float32'), self.createSimpleArray('int32')
x2o, y2o = x2.copy(), y2.copy()
td._store([x, x2], [y, y2], [w])
td.writeToFile("testfile.tdjctd")
td.addToFile("testfile.tdjctd")
td2 = TrainData()
td2._store([xo, x2o], [yo, y2o], [wo])
td2.append(td)
td.readFromFile("testfile.tdjctd")
os.system('rm -f testfile.tdjctd')
self.assertEqual(td, td2)