def test_kisa2 () :
from ostap.fitting.selectors import SelectorWithVars, Variable
variables = [
Variable ( 'mass' , 'mass(mu+mu-)' , 3.09 , 3.11 ) ,
Variable ( 'c2dtf' , 'chi2(dtf)' , -1 , 10 ) ,
Variable ( 'mass2' , 'mass(mu+mu-)' , 1 , 5 , MASS() ) ,
Variable ( 'mass3' , 'mass(mu+mu-)' , 1 , 5 , 'mass' )
]
from sys import version_info as python_version
if python_version.major < 3 :
variables.append ( Variable ( 'mass1' , 'mass(mu+mu-)' , 2 , 4 , lambda s : s.mass ) )
ppservers = () ## 'lxplus051' , )
## ppservers = 'auto'
nf = len ( data.files )
nf //= 40
nf += 1
nf = min ( nf , 25 )
with timing('%d files in sequence %s' % ( nf , len( data.chain ) ) ) :
selector = SelectorWithVars (
variables = variables ,
selection = '2<=mass && mass<4 && 0<=c2dtf && c2dtf<5' ,
silence = False
)
chain = data.chain[:nf]
st = chain.process ( selector , silent = False , shortcut = True )
ds = selector.data
del selector
logger.info ( 'Dataset:\n%s' % ds )
with timing('%s files in parallel %s' % ( len ( data.files ) , len( data.chain ) ) ) :
selector = SelectorWithVars (
variables = variables ,
selection = '2<=mass && mass<4 && 0<=c2dtf && c2dtf<5' ,
silence = True
)
st = data.chain.pprocess ( selector ,
silent = False ,
chunk_size = -1 ,
max_files = 1 ,
ppservers = ppservers )
ds = selector.data
del selector
logger.info ( 'Dataset:\n%s' % ds )
from ostap.fitting.selectors import SelectorWithVars, Variable
import ostap.parallel.parallel_fill
# =============================================================================
## configuration of reweighting
weightings = (
## variable address in DB
Weight.Var('x', 'x-reweight'),
Weight.Var('y', 'y-reweight'),
Weight.Var(('x', 'y'), '2D-reweight'),
)
# =============================================================================
## variables to be used in MC-dataset
variables = [
Variable('x', 'x-var', 0, 20),
Variable('y', 'y-var', 0, 15),
]
selector = SelectorWithVars(variables,
'0<x && x<20 && 0<y && y<20',
silence=True)
mctree.process(selector, silent=True)
mcds_ = selector.data ## dataset
# =============================================================================
## start reweighting iterations:
for iter in range(1, maxIter + 1):
logger.info(allright('Reweighting iteration %d ' % iter))
with timing('Prepare MC-dataset:', logger=logger):
# =========================================================================
from ostap.fitting.selectors import SelectorWithVars, Variable
## start iterations:
for iter in range ( 0 , maxIter ) :
weightings = (
## variable address in DB
Weight.Var ( lambda s : s.x , 'x-reweight' ) ,
Weight.Var ( lambda s : s.y , 'y-reweight' ) ,
Weight.Var ( lambda s : (s.x,s.y) , '2D-reweight' ) ,
)
weighter = Weight( dbname , weightings )
## variables to be used in MC-dataset
variables = [
Variable( 'x' , 'x-var' , 0 , 20 , lambda s : s.x ) ,
Variable( 'y' , 'y-var' , 0 , 15 , lambda s : s.y ) ,
Variable( 'weight' , 'weight' , accessor = weighter )
]
#
## create new "weighted" mcdataset
#
selector = SelectorWithVars (
variables ,
'0<x && x<20 && 0<y && y<20'
)
mctree.process ( selector )
mcds = selector.data ## new reweighted dataset
os.remove ( trainer.output_file )
# =============================================================================
## Use trained TMVA
# There are two alternatives
# - usage of TMVA Reader : it can be rather slow,
# but it is very flexible and powerful with respect to variable transformations
# - addTMVAResponse function : it is less flexible, but very CPU efficient
# =============================================================================
## prepare dataset with TMVA result
from ostap.fitting.selectors import SelectorWithVars, Variable
## 1) Book RooDataset
variables = [
Variable( 'var1' , 'variable#1' ) ,
Variable( 'var2' , 'variable#2' ) ,
Variable( 'var3' , 'variable#3' ) ,
]
## 2) create TMVA reader
from ostap.tools.tmva import Reader
reader = Reader( 'MyMLP' ,
variables = [ ('var1' , lambda s : s.var1 ) ,
('var2' , lambda s : s.var2 ) ,
('var3' , lambda s : s.var3 ) ] ,
weights_files = tar_file )
methods = reader.methods[:]
## # =============================================================================
#
from ostap.tools.reweight import Weight, makeWeights, WeightingPlot
from ostap.fitting.selectors import SelectorWithVars, Variable
## start iterations:
for iter in range ( 0 , maxIter ) :
weighting = (
## variable address in DB
Weight.Var( accessor = lambda s : s.x , address = 'x-reweight' ) ,
)
weighter = Weight( dbname , weighting )
## variables to be used in MC-dataset
variables = [
Variable ( 'pt_x' , 'pt_x' , 0 , 100 , lambda s : s.x ) ,
Variable ( 'weight' , 'weight' , accessor = weighter )
]
#
## create new "weighted" mcdataset
#
selector = SelectorWithVars (
variables ,
'0<x && x<100 '
)
mctree.process ( selector )
mcds = selector.data ## new reweighted dataset
#
#
from ostap.tools.reweight import Weight, makeWeights, WeightingPlot
from ostap.fitting.selectors import SelectorWithVars, Variable
## start iterations:
for iter in range(0, maxIter):
weighting = (
## variable address in DB
Weight.Var('x', address='x-reweight'), )
weighter = Weight(dbname, weighting)
## variables to be used in MC-dataset
variables = [
## Variable ( 'x' , 'x-variable' , 0 , 100 , lambda s : s.x ) ,
Variable('x', 'x-variable', 0, 100),
Variable('weight', 'weight', accessor=weighter)
]
#
## create new "weighted" mcdataset
#
selector = SelectorWithVars(variables, '0<x && x<100 ')
mctree.pprocess(selector, chunk_size=len(mctree) // 20)
##mctree.process ( selector )
mcds = selector.data ## new reweighted dataset
#
## update weights
#
for iter in range ( 0 , maxIter ) :
weightings = (
## variable address in DB
#Weight.Var ( lambda s : s.x , 'x-reweight' ) ,
#Weight.Var ( lambda s : s.y , 'y-reweight' ) ,
#Weight.Var ( lambda s : (s.x,s.y) , '2D-reweight' ) ,
Weight.Var ( 'x' , 'x-reweight' ) ,
Weight.Var ( 'y' , 'y-reweight' ) ,
Weight.Var ( ('x','y') , '2D-reweight' ) ,
)
weighter = Weight( dbname , weightings )
## variables to be used in MC-dataset
variables = [
Variable ( 'x' , 'x-var' , 0 , 20 ) ,
Variable ( 'y' , 'y-var' , 0 , 15 ) ,
Variable ( 'weight' , 'weight' , accessor = weighter )
]
#
## create new "weighted" mcdataset
#
selector = SelectorWithVars (
variables ,
'0<x && x<20 && 0<y && y<20'
)
mctree.pprocess ( selector , chunk_size = len(mctree) // 20 )
mcds = selector.data ## new reweighted dataset
else:
logger.info('Existing weights DBASE will be used')
#
## make reweighting iterations
#
from ostap.tools.reweight import Weight, makeWeights, WeightingPlot, W2Data
from ostap.fitting.selectors import SelectorWithVars, Variable
import ostap.parallel.parallel_fill
# =============================================================================
## weighting configuration: ## variable address in DB
weighting = (Weight.Var('x', address='x-reweight'), )
# ============================================================================
## variables to be used in MC-dataset
variables = [Variable('x', 'x-variable', 0, 100)]
# =============================================================================
## start iterations:
for iter in range(1, maxIter + 1):
logger.info(allright('Reweighting iteration %d ' % iter))
with timing('Prepare MC-dataset:', logger=logger):
# =============================================================================
## 0) The weighter object
weighter = Weight(dbname, weighting)
# ===============================================================================
## 1a) create mcdataset
methods = reader.methods[:]
# =============================================================================
## 1') few trivial tests: use the methods/reader as simple function
for m in methods :
method = reader[m]
## response = [ method ( i , 1.1 , 0.8 , 0.3 ) for i in range ( reader.N ) ]
response = method.stat ( 1.1 , 0.8 , 0.3 )
logger.info ( 'Simple test: method %10s,response %s' % ( m , response ) )
del method
# =============================================================================
from ostap.fitting.selectors import SelectorWithVars, Variable
## 2) Book RooDataset
variables = [
Variable ( 'var1' , 'variable#1' ) ,
Variable ( 'var2' , 'variable#2' ) ,
Variable ( 'var3' , 'variable#3' ) ,
## extra: needed for addChoppingResponse
Variable ( 'evt' , 'event' ) ,
Variable ( 'run' , 'run' ) ,
## extra: needed for cross-checks
Variable ( 'cat' , 'category' , accessor = category ) ,
]
## 3) declare/add TMVA variables
for m in methods :
variables += [ Variable ( 'tmva_%s' % m , 'TMVA(%s)' % m , accessor = reader[m] ) ]
## 4) Run Ostap to fill RooDataSet
from ostap.fitting.selectors import SelectorWithVars
## Use trained TMVA/Chopping
# There are two alternatives
# - usage of TMVA/Chopper Reader : it can be rather slow,
# but it is very flexible and powerful with respect to variable transformations
# - addChoppingResponse function : it is less flexible, but very CPU efficient
# =============================================================================
## category function
category = lambda s: int(s.evt * 137 + 813 * s.run) % N
## prepare dataset with TMVA/Chopping result
from ostap.fitting.selectors import SelectorWithVars, Variable
## 1) Book RooDataset
variables = [
Variable('var1', 'variable#1'),
Variable('var2', 'variable#2'),
Variable('var3', 'variable#3'),
## extra: needed for addChoppingResponse
Variable('evt', 'event'),
Variable('run', 'run'),
## extra: needed for cross-checks
Variable('cat', 'category', accessor=category),
]
## 2) create TMVA/Chopping reader
from ostap.tools.chopping import Reader
reader = Reader(
N=N, ## number of categories
categoryfunc=category, ## category