The module has been developed and used with great success in ``Kali, framework for fine calibration of LHCb Electormagnetic Calorimeter'' """ # ============================================================================= __author__ = "Vanya BELYAEV [email protected]" __date__ = "2010-04-30" __version__ = "$Revision$" # ============================================================================= __all__ = ( 'CompressShelf', ## The DB-itself ) # ============================================================================= from ostap.logger.logger import getLogger if '__main__' == __name__: logger = getLogger('ostap.io.compress_shelve') else: logger = getLogger(__name__) # ============================================================================= PROTOCOL = -1 ENCODING = 'utf-8' # ============================================================================== import os, sys, abc, shelve, shutil, glob, datetime, collections from sys import version_info as python_version from ostap.io.dbase import dbopen, whichdb, Item from ostap.core.meta_info import meta_info # ============================================================================= _modes_ = { # ========================================================================= # 'r' Open existing database for reading only # 'w' Open existing database for reading and writing # 'c' Open database for reading and writing, creating it if it doesn't exist
__date__ = "2020-05-11"
__all__ = (
##
'TrPDF', ## 1D-transformed PDF
##
)
# =============================================================================
import ROOT
import ostap.fitting.roofit
from ostap.fitting.roofuncs import var_mul
from ostap.fitting.funbasic import FUNC
from ostap.fitting.basic import PDF
from ostap.fitting.fit2d import PDF2
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__: logger = getLogger('ostap.fitting.transform_pdf')
else: logger = getLogger(__name__)
# =============================================================================
## @class TrPDF
# PDF of the transformed variable.
# - e.g. gaussian in log10 scale:
# @code
#
# x = ROOT.RooRealVar ( 'x' ,'' ,1,100000 ) ## original/old variable
# lx = ROOT.RooRealVar ( 'lx' ,'log10(x)',0,5 ) ## new_variable
# LX = Fun1D( lx , lx )
# NX = 10 ** LX ## old variable as a function of new variable
#
'lcm', ## find the least common multiple
#
'interpolate', ## construct Bernstein interpolant
#
'generate', ## generate random numbers
'shoot', ## generate random numbers
)
# =============================================================================
import ROOT, math
from ostap.core.core import cpp, Ostap, funID
from ostap.math.base import iszero, isequal, signum
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__: logger = getLogger('ostap.math.bernstein')
else: logger = getLogger(__name__)
# =============================================================================
## short name
Bernstein = cpp.Ostap.Math.Bernstein
# =============================================================================
## get control polygon for Bernstein polynomial
def control_polygon(bp):
"""Get control polygon for Bernstein polynomial
>>> bernstein = ...
>>> cp = bernstein.control_polygon ()
>>> cp = control_polygon( bernstein ) ## ditto
"""
return Ostap.Math.control_polygon(bp.bernstein())
def test_components_2():
logger = getLogger('test_components_2')
## make simple test mass
mass = ROOT.RooRealVar('test_mass', 'Some test mass', 0, 10)
## book very simple data set
varset = ROOT.RooArgSet(mass)
dataset = ROOT.RooDataSet(dsID(), 'Test Data set', varset)
mmin, mmax = mass.minmax()
## two gaussinan signal cmppnent
N1 = 5000
m1 = VE(5, 0.2**2)
N2 = 5000
m2 = VE(3, 0.3**2)
## background exponential components
taub = 5.0
NB = 5000
## generate 1st signal
for i in range(0, N1):
m = m1.gauss()
while not mmin < m < mmax:
m = m1.gauss()
mass.value = m
dataset.add(varset)
## generate 2nd signal
for i in range(0, N2):
m = m2.gauss()
while not mmin < m < mmax:
m = m2.gauss()
mass.value = m
dataset.add(varset)
## generate background
for i in range(0, NB):
m = random.expovariate(1. / taub)
while not mmin < m < mmax:
m = random.expovariate(1. / taub)
mass.value = m
dataset.add(varset)
logger.info("Original dataset\n%s" % dataset.table(prefix='# '))
signal1 = Models.Gauss_pdf('G1',
xvar=mass,
mean=(5, 4, 6),
sigma=(0.2, 0.1, 0.3))
signal2 = Models.Gauss_pdf('G2',
xvar=mass,
mean=(3, 2, 4),
sigma=(0.3, 0.2, 0.4))
S1 = ROOT.RooRealVar('S1', '1st signal yeild', N1, 1, 10000)
ratio = ROOT.RooRealVar('R', 'ratio of S2 to S1', 1.0 * N2 / N1, 0.05, 100)
S2 = signal1.vars_multiply(S1, ratio, name='S2', title='2ns signal yield')
model = Models.Fit1D(signals=[signal1, signal2], background=1, S=(S1, S2))
model.B = NB
signal1.mean.fix(m1.value())
signal1.sigma.fix(m1.error())
signal2.mean.fix(m2.value())
signal2.sigma.fix(m2.error())
model.fitTo(dataset, silent=True)
signal1.mean.release()
signal1.sigma.release()
signal2.mean.release()
signal2.sigma.release()
with use_canvas('test_components_2'):
model.fitTo(dataset, silent=True)
r, f = model.fitTo(dataset, silent=True, draw=True, nbins=50)
logger.info("Mass fit : fit results\n%s" %
r.table(title='Mass fit', prefix='# '))
r, f = model.fitTo(dataset,
silent=True,
draw=True,
nbins=50,
minos=('R', 'S1'))
logger.info("Mass fit : fit results\n%s" %
r.table(title='Mass fit', prefix='# '))
"pyfun_data", ## ditto, but as fnuction
'FuncFormula', ## simple wrapper over TTreeFormula/Ostap::Formula
'FuncRooFormula', ## simple wrapper over RooFormulaVar
'FuncTH1', ## TH1-based Tree-function
'FuncTH2', ## TH2-based Tree-function
'FuncTH3', ## TH3-based Tree-function
)
# =============================================================================
import ROOT
from ostap.core.core import Ostap, valid_pointer
from ostap.core.meta_info import old_PyROOT
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__: logger = getLogger('ostap.trees.funcs')
else: logger = getLogger(__name__)
# =============================================================================
## @class FuncTree
# Helper class to implement "TTree-function"
# @see Ostap::Functions::PyFuncTree
class FuncTree(Ostap.Functions.PyFuncTree):
"""Helper class to implement ``TTree-function'' in python
"""
def __init__(self, tree=None):
## initialize the base class
if tree is None: tree = ROOT.nullptr
##
if old_PyROOT: super(FuncTree, self).__init__(self, tree)
""" # ============================================================================= __author__ = "Vanya BELYAEV [email protected]" __date__ = "2009-09-12" __version__ = "" # ============================================================================= __all__ = ( 'C2FIT' , ## simple chi2-fit ) # ============================================================================= import ROOT, cppyy # ============================================================================= # logging # ============================================================================= from ostap.logger.logger import getLogger if '__main__' == __name__ : logger = getLogger ( 'ostap.fitting.chi2fit' ) else : logger = getLogger ( __name__ ) # ============================================================================= from ostap.math.ve import VE,cpp cpp = cppyy.gbl Ostap = cpp.Ostap C2FIT = Ostap.Math.Chi2Fit C2FIT . __str__ = lambda s : s.toString () C2FIT . __repr__ = lambda s : s.toString () # ============================================================================= ## chi2-probabilty def _c2_prob_ ( s ) : """Chi2 probabiilty >>> r = h.hfit ( ... ) >>> r.Prob()
# Copyright (c) Ostap developpers.
# =============================================================================
## @file ostap/math/tests/test_math_primes.py
# Test module for the file ostap/math/primes.py
# =============================================================================
""" Test module for ostap/math/primes.py
It tests local implementation of prime numbers sieve
"""
# =============================================================================
from __future__ import print_function
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__: logger = getLogger('tests_math_primes')
else: logger = getLogger(__name__)
# =============================================================================
from ostap.math.primes import primes, Primes
def test_primes1():
p = primes(1000)
logger.info('Primes:\n%s' % p)
def test_primes2():
p = Primes(10000)
from __future__ import print_function
# =============================================================================
import ROOT, random
import ostap.fitting.roofit
import ostap.fitting.models as Models
from ostap.core.core import Ostap, std, VE, dsID
from ostap.logger.utils import rooSilent
import ostap.io.zipshelve as DBASE
from ostap.utils.timing import timing
from builtins import range
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ or '__builtin__' == __name__ :
logger = getLogger ( 'test_fitting_models6_2D' )
else :
logger = getLogger ( __name__ )
# =============================================================================
## make simple test mass
m_x = ROOT.RooRealVar ( 'mass_x' , 'Some test mass(X)' , 3 , 3.2 )
m_y = ROOT.RooRealVar ( 'mass_y' , 'Some test mass(Y)' , 3 , 3.2 )
## book very simple data set
varset = ROOT.RooArgSet ( m_x , m_y )
dataset = ROOT.RooDataSet ( dsID() , 'Test Data set-1' , varset )
m = VE(3.100,0.015**2)
N_ss = 5000
N_sb = 500
'Barycentric', ## polynomian true Barycentric interpolant
'FloaterHormann', ## rational Floater-Hormann interpolant
)
# =============================================================================
import ROOT, math, sys, abc
from array import array
from builtins import range
from ostap.core.core import cpp, Ostap
from ostap.core.ostap_types import (is_integer, sequence_types, integer_types,
dictlike_types)
from ostap.math.base import iszero, isequal, doubles
from ostap.utils.utils import vrange
import ostap.math.reduce
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__: logger = getLogger('ostap.math.interpolation')
else: logger = getLogger(__name__)
# =============================================================================
if (3, 3) <= sys.version_info: from collections.abc import Iterable, Mapping
else: from collections import Iterable, Mapping
# =============================================================================
# =============================================================================
## Interpolation abscissas
# =============================================================================
# =============================================================================
## printout for the interpolation abscissas
# @code
""" # ============================================================================= __version__ = "$Revision$" __author__ = "Vanya BELYAEV [email protected]" __date__ = "2012-10-16" # ============================================================================= __all__ = ( 'getLumi' , ## get the lumi ) # ============================================================================= import ROOT # ============================================================================= # logging # ============================================================================= from ostap.logger.logger import getLogger if '__main__' == __name__ : logger = getLogger ( 'ostap.contrib.lhcb.lumi' ) else : logger = getLogger ( __name__ ) # ============================================================================= ## get luminosity from Lumi tuple # # @param data (INPUT) tree, chain, file, filename or sequence # @return the luminosity # @attention Linear addition of uncertainties is used here # # @code # # >>> l1 = getLumi ( 'myfile.root' ) # >>> l2 = getLumi ( tree ) # >>> l3 = getLumi ( chain ) # >>> l4 = getLumi ( file ) # >>> l5 = getLumi ( [ any sequence of above ] )
# =============================================================================
__author__ = "Ostap developers"
__all__ = () ## nothing to import
# =============================================================================
import ROOT, random
import ostap.fitting.roofit
import ostap.fitting.models as Models
from ostap.core.core import dsID
from ostap.utils.timing import timing
from ostap.logger.utils import rooSilent
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ or '__builtin__' == __name__:
logger = getLogger('test_fitting_simfit3')
else:
logger = getLogger(__name__)
# =============================================================================
## make simple test mass
mass1 = ROOT.RooRealVar('test_mass1', 'Some test mass', 0, 10)
mass2 = ROOT.RooRealVar('test_mass2', 'Some test mass', 0, 20)
mass3 = ROOT.RooRealVar('test_mass3', 'Some test mass', 0, 20)
## book very simple data set:
varset1 = ROOT.RooArgSet(mass1)
dataset1 = ROOT.RooDataSet(dsID(), 'Test Data set-1', varset1)
## book very simple data set:
varset2 = ROOT.RooArgSet(mass2, mass3)
dataset2 = ROOT.RooDataSet(dsID(), 'Test Data set-2', varset2)
def test_fitting_components3_3D () :
logger = getLogger( 'test_fitting_components3_3D' )
## make simple test mass
m_x = ROOT.RooRealVar ( 'mass_x' , 'Some test mass(X)' , 0 , 10 )
m_y = ROOT.RooRealVar ( 'mass_y' , 'Some test mass(Y)' , 0 , 10 )
m_z = ROOT.RooRealVar ( 'mass_z' , 'Some test mass(z)' , 0 , 10 )
## book very simple data set
varset = ROOT.RooArgSet ( m_x , m_y,m_z )
dataset = ROOT.RooDataSet ( dsID() , 'Test Data set-1' , varset )
m1 = VE ( 3 , 0.10**2 )
m2 = VE ( 7 , 0.20**2 )
## fill it with three gausissians, 5k events each
N_sss = 5000
N_ssb = 500
N_sbs = N_ssb
N_bss = N_ssb
N_bbs = 100
N_bsb = N_bbs
N_sbb = N_bbs
N_bbb = 500
random.seed(0)
## fill it : 5000 events Gauss * Gauss *Gauss
for m in (m1,m2) :
## S x S x S
for i in range ( N_sss ) :
m_x.value = m.gauss()
m_y.value = m.gauss()
m_z.value = m.gauss()
dataset.add ( varset )
## S x S x B
for i in range(0,N_bss) :
m_x.value = m.gauss()
m_y.value = m.gauss()
m_z.value = random.uniform ( *m_z.minmax() )
dataset.add ( varset )
## S x B x S
for i in range ( N_ssb ) :
m_x.value = m.gauss()
m_y.value = random.uniform ( *m_y.minmax() )
m_z.value = m.gauss()
dataset.add ( varset )
## B x S x S
for i in range ( N_sbs ) :
m_x.value = random.uniform ( *m_x.minmax() )
m_y.value = m.gauss()
m_z.value = m.gauss()
dataset.add ( varset )
## B x B X S
for i in range ( N_sbb ) :
m_x.value = random.uniform ( *m_x.minmax() )
m_y.value = random.uniform ( *m_y.minmax() )
m_z.value = m.gauss()
dataset.add ( varset )
## B x S x B
for i in range ( N_bsb ) :
m_x.value = random.uniform ( *m_x.minmax() )
m_y.value = m.gauss()
m_z.value = random.uniform ( *m_y.minmax() )
dataset.add ( varset )
## S x B x B
for i in range ( N_sbb ) :
m_x.value = m.gauss()
m_y.value = random.uniform ( *m_y.minmax() )
m_z.value = random.uniform ( *m_y.minmax() )
dataset.add ( varset )
## B x B x B
for i in range ( N_bbb ) :
m_x.value = random.uniform ( *m_x.minmax() )
m_y.value = random.uniform ( *m_y.minmax() )
m_z.value = random.uniform ( *m_y.minmax() )
dataset.add ( varset )
logger.info ('Dataset:\n%s' % dataset.table ( prefix = '# ') )
## various fit components
signal_x1 = Models.Gauss_pdf ( 'G1x' , xvar = m_x , mean = m1.value() , sigma = m1.error() )
signal_y1 = signal_x1.clone ( name='G1y' , xvar = m_y )
signal_z1 = signal_x1.clone ( name='G1z' , xvar = m_z )
bkg_x = make_bkg ( -1 , 'Bx' , m_x )
bkg_y = bkg_x.clone ( name= 'By' , xvar =m_y )
bkg_z = bkg_x.clone ( name='Bz' , xvar =m_z )
signal_x2 = Models.Gauss_pdf ( name='G2x' , xvar = m_x , mean = m2.value() , sigma = m2.error() )
signal_y2 = signal_x2.clone ( name='G2y' , xvar = m_y )
signal_z2 = signal_x2.clone ( name='G2z' , xvar = m_z )
# S(x) * S(y) component
ss_cmp = signal_x2 * signal_y2
# S(x) * B(y) component
sb_cmp = signal_x2 * bkg_y
# B(x) * S(y) component
bs_cmp = bkg_x * signal_y2
# B(x) * B(y) component
bb_cmp = bkg_x * bkg_y
# S(x)*S(y)*S(z) component
sss_cmp = ss_cmp * signal_z2
# S(x) * B(y) * S(z) + B(x) * S(y) * S(z) + S(x) * S(y) * B(z) component
ssb_ = ss_cmp * bkg_z
sbs_ = sb_cmp * signal_z2
bss_ = bs_cmp * signal_z2
ssb_cmp = ssb_ + ( sbs_ , bss_ )
# S(x)*B(y)*B(z) + B(x)*S(y)*B(z) + B(x)*B(y)*S(z) component
sbb_ = sb_cmp * bkg_z
bsb_ = bs_cmp * bkg_z
bbs_ = bb_cmp * signal_z2
bbs_cmp = sbb_ + ( bsb_ , bbs_ )
logger.info ('Test multi-component 3d Sym fit')
model = Models.Fit3DMix (
name = 'fitSym_comp',
signal_x = signal_x1,
signal_y = signal_y1,
signal_z = signal_z1,
bkg_1x = bkg_x ,
bkg_1y= bkg_y,
bkg_2x = 'clone' ,
bkg_2y= 'clone',
components = [ sss_cmp , ssb_cmp , bbs_cmp ]
)
## components
model.SSS = N_sss
model.SSB = N_ssb * 3
model.SBB = N_sbb * 3
model.BBB = N_bbb
model.C = N_sss, N_ssb * 3 , N_sbb * 3
r = model.fitTo ( dataset , silent = True )
r = model.fitTo ( dataset , silent = True )
r = model.fitTo ( dataset , silent = True )
with use_canvas ( 'test_fitting_components3_3D' ) :
with wait ( after = 2 ) : model.draw1 ( dataset )
with wait ( after = 2 ) : model.draw2 ( dataset )
with wait ( after = 2 ) : model.draw3 ( dataset )
logger.info ( 'Model %s Fit result\n%s ' % ( model.name , r.table (prefix = '# ') ) )
from __future__ import print_function
# =============================================================================
import ROOT, random
import ostap.fitting.roofit
import ostap.fitting.models as Models
from ostap.core.core import Ostap, std, VE, dsID
from ostap.logger.utils import rooSilent
import ostap.io.zipshelve as DBASE
from ostap.utils.timing import timing
from builtins import range
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ or '__builtin__' == __name__:
logger = getLogger('test_fitting_breitwigner')
else:
logger = getLogger(__name__)
# =============================================================================
## make simple test mass
GeV = 1.0
MeV = 0.001 * GeV
m_pi = 139 * MeV
m_rho = 770 * MeV
g_rho = 150 * MeV
def test_breitwigner_rho():
# @date 2011-06-07 # ============================================================================= """Decoration of RooCmdArg obejcts """ # ============================================================================= __version__ = "$Revision$" __author__ = "Vanya BELYAEV [email protected]" __date__ = "2011-06-07" __all__ = () # ============================================================================= import ROOT # ============================================================================= # logging # ============================================================================= from ostap.logger.logger import getLogger, allright, attention if '__main__' == __name__: logger = getLogger('ostap.fitting.roocmdarg') else: logger = getLogger(__name__) # ============================================================================= ## print RooCmdArg object def _rca_print_(self): """Print RooCmdArg object """ name = self.GetName() ## RooAbsReal::plotOn arguments if 'DrawOption' == name: return "DrawOptions('%s')" % self.getString(0) elif 'SliceVars' == name: return "Slice({.})" elif 'SliceCat' == name: return "Slice({.})" elif 'Project' == name: return "Project({.})"
# =============================================================================
""" Test module for ostap/histos/param.py
- It tests parameterisations of histograms
"""
# =============================================================================
__author__ = "Ostap developers"
__all__ = () ## nothing to import
# =============================================================================
import ROOT, random, ostap.histos.param, ostap.histos.histos, ostap.fitting.funcs
from builtins import range
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ or '__builtin__' == __name__ :
logger = getLogger ( 'ostap.test_histos_parameterisation2' )
else :
logger = getLogger ( __name__ )
# =============================================================================
logger.info ( 'Test for histogram parameterisation')
# =============================================================================
try :
import scipy
except ImportError :
scipy = None
# =============================================================================
from ostap.histos.param import legendre_sum, chebyshev_sum
from ostap.core.core import hID, fID
from ostap.utils.timing import timing
# @date 2016-02-23 # ============================================================================= """Useful utilities for multiprocessing and parallel processing for Ostap - Helper class to start remote ppserver (for parallel python) """ # ============================================================================= __version__ = '$Revision$' __author__ = 'Vanya BELYAEV [email protected]' __date__ = '2016-02-23' __all__ = ( 'ppServer', ## helper class to start remote ppserver (for parallel python) 'show_tunnels', ## show the table of tunnels ) # ============================================================================= from ostap.logger.logger import getLogger, keepLevel, enabledVerbose if '__main__' == __name__: logger = getLogger('ostap.paralllel.pptunnel') else: logger = getLogger(__name__) # ============================================================================= import sys, os, time import pathos.core as PC import pathos.secure as PS # ============================================================================= ## @class ppServer # Helper class that starts <code>ppserver</code> on remote site # and makes SSH tunnel # @attention Password-less SSH-connection is required! # @code # pps = ppServer('lxplus009.cern.ch') # print pps.local, pps.remote
__author__ = "Vanya BELYAEV [email protected]" __date__ = "2011-06-07" __all__ = ( 'ROOTCWD', ## context manager to keep Current Directory 'open_mode', ## decode open-mode for ROOT-files 'open', ## just for completness 'REOPEN', ## context manager to <code>ROOT.TFileReOpen('UPDATE')</code> ) # ============================================================================= import ROOT, os, cppyy ## attention here!! cpp = cppyy.gbl # ============================================================================= # logging # ============================================================================= from ostap.logger.logger import getLogger if '__main__' == __name__: logger = getLogger('ostap.io.root_file') else: logger = getLogger(__name__) # ============================================================================= logger.debug('Some useful decorations for TFile objects') # ============================================================================== ## context manager to preserve current directory (rather confusing stuff in ROOT) from ostap.core.core import ROOTCWD # =============================================================================== ## write the (T)object to ROOT-file/directory # @code # histo1 = ... # histo2 = ... # rfile['MyHisto' ] = histo1 # rfile['MyDir/MyHisto'] = histo2
def logger(self):
"""``logger'' : local logger"""
if not self.__logger:
self.__logger = getLogger("pptunnel:%s" % self.remote_host)
return self.__logger
__all__ = (
'scalar_minimize' , ## local copy of minimize_scalar from scipy
'minimize_scalar' , ## the main entry
## helper functions:
'sp_minimum_1D' ,
'sp_maximum_1D' ,
'sp_minimum_2D' ,
'sp_maximum_2D' ,
'sp_minimum_3D' ,
'sp_maximum_3D' ,
)
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ : logger = getLogger ( 'ostap.math.minimize' )
else : logger = getLogger ( __name__ )
# =============================================================================
import math, warnings
from math import sqrt
try :
import numpy
import numpy as np
_epsilon = math.sqrt(numpy.finfo(float).eps)
except ImportError :
class numpy(object) :
@staticmethod
def abs ( value ) : return abs ( value )
@staticmethod
def size ( value ) : return 1
import sys
def __init__(self,
remote,
environment='',
script=None,
profile=None,
secret=None,
timeout=-1,
silent=True):
self.__session = None
self.__pid = 0
self.__active = False
self.__silent = True if silent else False
## split user@remote
remote_user, at, remote = remote.rpartition('@')
import socket
remote = socket.getfqdn(remote).lower()
## ## merge user@remote
## if remote_user and at :
## import getpass
## local_user = getpass.getuser()
## if remote_user.lower() != local_user.lower() :
## remote = ''.join ( ( remote_user, at , remote ) )
remote = ''.join((remote_user, at, remote))
self.__logger = getLogger('pptunnel:%s' % remote)
self.__remote_host = remote
## SSH tunnel
self.__tunnel = PS.Tunnel('SSH-tunnel %s' % self.remote_host)
self.tunnel.connect(self.remote_host)
self.logger.debug('Create SSH tunnel : %s' % self.tunnel)
self.__local_port = self.tunnel._lport
self.__remote_port = self.tunnel._rport
## SSH session
self.__session = PS.Pipe('SSH-session', host=self.remote_host)
self.logger.debug('Create SSH session: %s' % self.session)
# import logging
# verbose = logging.root.manager.disable <= logging.DEBUG
# self.session.verbose = verbose
# self.tunnel .verbose = verbose
# del logging
with keepLevel():
# =================================================================
# Get configuration information for the given remote host
if (not environment) or (not script) or (not profile):
from ostap.parallel.utils import get_remote_conf
e, s, p = get_remote_conf(remote)
if e and not environment: environment = e
if s and not script: script = s
if p and not profile: profile = p
if script:
if os.path.exists(script) and os.path.isfile(script): pass
else:
self.logger.error("The script %s does not exist, skip it!" %
script)
script = None
## temporary directory on remote host
self.__tmpdir = None
if environment or script or profile or (not self.silent) or True:
self.logger.verbose('Creating remote temporary directory')
## self.session ( command = 'TMPDIR=${TMPDIR} mktemp -d -t $(whoami)-ostap-pathos-$(date +%Y-%b-%d)-XXXXXXXXX' )
self.session(
command=
'mktemp -d -t $(whoami)-ostap-pathos-$(date +%Y-%b-%d)-XXXXXXXXX'
)
self.session.launch()
r = self.session.response()
if r and 1 < len(r):
self.__tmpdir = r[:-1]
self.logger.debug('Created remote temporary directory %s' %
self.__tmpdir)
else:
self.logger.error('Cannot create remote temporary directory')
self.__environment = None
if environment:
##
self.logger.verbose("Processing the environment:\n%s" %
environment)
##
import ostap.utils.cleanup as CU
tmpfile = CU.CleanUp.tempfile(prefix='ostap-env-', suffix='.sh')
with open(tmpfile, 'w') as tf:
tf.write(environment)
tf.write('\n')
copier = PS.Copier('SSH-copier')
destination = "%s:%s" % (self.__remote_host, self.__tmpdir)
copier(source=tmpfile, destination=destination)
copier.launch()
r = copier.response()
if r: self.logger.error('SCP: response from %s : %s' % (copier, r))
del copier
self.__environment = os.path.join(self.__tmpdir,
os.path.basename(tmpfile))
self.logger.debug('Environment is copied to %s:%s ' %
(self.remote_host, self.environment))
self.__script = None
if script and os.path.exists(script) and os.path.isfile(script):
self.logger.verbose("Processing the script %s" % scriot)
with open(script, 'r') as f:
for line in f:
if not line: continue
self.logger.verbose(line[:-1])
##
copier = PS.Copier('SSH-copier')
destination = "%s:%s" % (self.__remote_host, self.__remote_tmpdir)
copier(source=script, destination=destination)
copier.launch()
r = copier.response()
if r: self.logger.error('SPC: response from %s : %s' % (copier, r))
del copier
self.__script = os.path.join(self.__tmpdir,
os.path.basename(script))
self.logger.debug('Script %s is copied to %s:%s' %
(script, self.remote_host, self.__script))
self.__profile = None
if profile:
self.logger.verbose("Processing the profile %s" % profile)
self.session(command='[ -f %s ] && echo "1" || echo "0" ' %
profile)
self.session.launch()
r = self.session.response()
try:
if int(r): self.__profile = profile
except:
pass
if self.__profile:
self.logger.debug("Profile %s:%s is found" %
(self.remote_host, profile))
if enabledVerbose():
copier = PS.Copier('SSH-copier')
import ostap.utils.cleanup as CU
print('BEFORE CREATION!')
tmpdir = CU.CleanUp.tempdir()
print('AFTER CREATION!', tmpdir)
source = "%s:%s" % (self.__remote_host, self.__profile)
copier(source=source, destination=tmpdir)
copier.launch()
r = copier.response()
if r:
self.logger.error('SPC: response from %s : %s' %
(copier, r))
del copier
local_profile = os.path.join(
tmpdir, os.path.basename(self.__profile))
with open(local_profile, 'r') as f:
for line in f:
if not line: continue
self.logger.verbose(line[:-1])
else:
self.logger.error("Profile %s:%s is NOT found" %
(self.remote_host, profile))
commands = []
if self.__profile: commands.append(' source %s ' % profile)
if self.__environment:
commands.append(' source %s ' % self.__environment)
if self.__script: commands.append(' source %s ' % self.__script)
## pp-server itself:
if secret and 1 < timeout:
commands.append('ppserver -p %-7d -s %s -t %d' %
(self.remote_port, secret, timeout))
pattern = '[P,p]ython *.*ppserver *-p *%d *-s *%s *-t *%d' % (
self.remote_port, secret, timeout)
elif secret:
commands.append('ppserver -p %-7d -s %s' %
(self.remote_port, secret))
pattern = '[P,p]ython *.*ppserver *-p *%d *-s *%s' % (
self.remote_port, secret)
elif 1 < timeout:
commands.append('ppserver -p %-7d -t %d' %
(self.remote_port, timeout))
pattern = '[P,p]ython *.*ppserver *-p *%d *-t *%d' % (
self.remote_port, timeout)
else:
commands.append('ppserver -p %-7d' % self.remote_port)
pattern = '[P,p]ython *.*ppserver *-p *%d' % self.remote_port
pid_file = "%s/ppserver.pid" % self.__tmpdir if self.__tmpdir else None
if pid_file:
commands[-1] += " -P %s" % pid_file
pattern += " *-P %s" % pid_file
command = ' && '.join(commands)
self.session(command=command, background=True, options='-f')
self.session.launch()
r = self.session.response()
if r:
self.logger.error('SERVER:response from %s : %s' %
(self.session, r))
self.logger.debug('Command launched "%s"' % command)
## try to pickup PID through the PID-file (could be more efficient)
if pid_file:
self.logger.verbose("Use PID-file %s " % pid_file)
time.sleep(0.5)
for i in range(10):
time.sleep(0.25)
command = 'cat %s' % pid_file
self.session.verbose = False
self.session(command=command)
self.session.launch()
response = self.session.response()
if not response: continue
try:
pid = int(response)
except ValueError:
pid = 0
if 0 < pid:
self.__pid = pid
break
## pickup PID via parsing
if not self.pid:
self.logger.verbose("Use regex '%s' to locate PID" % pattern)
time.sleep(0.5)
for i in range(20):
try:
time.sleep(0.25)
self.__pid = PC.getpid(pattern, self.remote_host)
except OSError:
pass
if 0 < self.pid: break
if not self.pid:
self.logger.error(
'Cannot acquire PID for remote ppserver at %s:%s' %
(self.remote_host, self.remote_port))
else:
self.logger.debug('PID for remote ppserver is %-6d ' % self.pid)
sys.stdout.flush() ## needed?
sys.stdin.flush() ## needed ?
if not self.silent:
self.logger.info('Tunnel: %6d -> %s:%s pid:%-6d' %
(self.local_port, self.remote_host,
self.remote_port, self.pid))
else:
self.logger.debug("Tunnel:%s" % self)
self.start()
__all__ = () ## nothing to import
# =============================================================================
import ROOT, random
import ostap.fitting.roofit
import ostap.fitting.models as Models
from ostap.core.core import cpp, VE, dsID
from ostap.logger.utils import rooSilent
from builtins import range
from ostap.utils.timing import timing
from ostap.plotting.canvas import use_canvas
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ or '__builtin__' == __name__:
logger = getLogger('test_fitting_components2')
else:
logger = getLogger(__name__)
# =============================================================================
def test_components_2():
logger = getLogger('test_components_2')
## make simple test mass
mass = ROOT.RooRealVar('test_mass', 'Some test mass', 0, 10)
## book very simple data set
varset = ROOT.RooArgSet(mass)
dataset = ROOT.RooDataSet(dsID(), 'Test Data set', varset)
# =============================================================================
## parse arguments
# =============================================================================
arguments = parse_args()
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if arguments.Color and not arguments.batch :
from ostap.logger.logger import make_colors
make_colors()
del make_colors
logger = getLogger( 'ostap' )
# =============================================================================
# =============================================================================
## suppress extra prints from logging
# =============================================================================
if arguments.Quiet :
import logging
logger.info ( '(silent) Interactive Ostap session (steroid-enhanced PyROOT)')
logging.disable ( logging.WARNING - 1 )
else:
import logging
level = logging.DEBUG-5 if arguments.Verbose else logging.INFO-1
from __future__ import print_function
# =============================================================================
__author__ = "Ostap developers"
__all__ = () ## nothing to import
# =============================================================================
import ROOT, random
import ostap.fitting.roofit
from ostap.core.core import VE, dsID
from builtins import range
from ostap.utils.timing import timing
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ or '__builtin__' == __name__:
logger = getLogger('test_fitting_resolutions')
else:
logger = getLogger(__name__)
# =============================================================================
## make simple test mass
mass = ROOT.RooRealVar('test_mass', 'Some test mass', -3, 3)
## book very simple data set
varset0 = ROOT.RooArgSet(mass)
dataset0 = ROOT.RooDataSet(dsID(), 'Test Data set-0', varset0)
mmin = mass.getMin()
mmax = mass.getMax()
## fill it
m1 = VE(0, 0.1**2)
# @date 2011-06-07 # ============================================================================= """Tiny decoration for ROOT.TF objects """ # ============================================================================= __version__ = "$Revision$" __author__ = "Vanya BELYAEV [email protected]" __date__ = "2011-06-07" __all__ = () ## nothing to import # ============================================================================= import ROOT # ============================================================================= # logging # ============================================================================= from ostap.logger.logger import getLogger if '__main__' == __name__: logger = getLogger('ostap.fitting.funcs') else: logger = getLogger(__name__) # ============================================================================= logger.debug('Tiny decoration for ROOT.TF objects') # ============================================================================= from ostap.core.core import cpp, Ostap, VE, funID # ============================================================================= # ============================================================================= ## Generate random tuple according to TF2 # @code # f2 = ... ## ROOT TF2 # x,y = f2.random() ## get random number # @endcode
# =============================================================================
import ROOT, random, math
import ostap.fitting.roofit
import ostap.fitting.models as Models
from ostap.core.core import cpp, VE, dsID, Ostap
from ostap.logger.utils import rooSilent
from ostap.fitting.efficiency import Efficiency1D
from ostap.utils.timing import timing
from ostap.plotting.canvas import use_canvas
from ostap.utils.utils import wait
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ or '__builtin__' == __name__:
logger = getLogger('test_fitting_efficiency')
else:
logger = getLogger(__name__)
# =============================================================================
## make
x = ROOT.RooRealVar('x', 'test', 0, 10)
xmin, xmax = x.minmax()
acc = ROOT.RooCategory('cut', 'cut')
acc.defineType('accept', 1)
acc.defineType('reject', 0)
varset = ROOT.RooArgSet(x, acc)
ds = ROOT.RooDataSet(dsID(), 'test data', varset)
eff0 = Models.Monotonic_pdf('E0', xvar=x, power=3, increasing=True)
eff0.phis = 3.1415 / 1, 3.1415 / 2, 3.1415 / 3
""" # ============================================================================= from __future__ import print_function # ============================================================================= __author__ = "Vanya BELYAEV [email protected]" __date__ = "2014-06-19" __version__ = "$Revision$" # ============================================================================= __all__ = ( 'SQLiteShelf', ## The DB-itself 'open', ## helper function to hide the actual DB 'tmpdb', ## helper function to create the temporary database ) # ============================================================================= from ostap.logger.logger import getLogger if '__main__' == __name__: logger = getLogger('ostap.io.sqliteshelve') else: logger = getLogger(__name__) # ============================================================================= import os, sys, zlib, collections, datetime import sqlite3 from ostap.io.sqlitedict import SqliteDict from ostap.io.dbase import Item from ostap.core.meta_info import meta_info # ============================================================================= try: from cPickle import Pickler, Unpickler, HIGHEST_PROTOCOL except ImportError: from pickle import Pickler, Unpickler, HIGHEST_PROTOCOL # ============================================================================= PROTOCOL = 2 # =============================================================================
time.sleep ( 0.02 )
print('Example 2: Dynamic Bar')
with ProgressBar(0, limit, mode='dynamic', char='-') as bar :
for i in range(limit+1):
bar += 1
time.sleep ( 0.02 )
for i in progress_bar( range(15000) , description = "Doing something ") :
time.sleep(0.001)
for i in running_bar( range(15000) , description = "Empty looping ") :
time.sleep(0.001)
# ==============================================================================
if __name__ == '__main__':
from ostap.logger.logger import getLogger
logger = getLogger('ostap.utils.progress_bar')
from ostap.utils.docme import docme
docme ( __name__ , logger = logger )
test_bars ()
logger.info ( 80*'*' )
# =============================================================================
# The END
# =============================================================================
def ls(self, pattern='', load=True):
"""List the available keys (patterns included).
Pattern matching is performed accoriding to
fnmatch/glob/shell rules [it is not regex!]
>>> db = ...
>>> db.ls() ## all keys
>>> db.ls ('*MC*')
"""
n = os.path.basename(self.filename)
ap = os.path.abspath(self.filename)
try:
fs = os.path.getsize(self.filename)
except:
fs = -1
if fs < 0: size = "???"
elif fs < 1024: size = str(fs)
elif fs < 1024 * 1024:
size = '%.2fkB' % (float(fs) / 1024)
elif fs < 1024 * 1024 * 1024:
size = '%.2fMB' % (float(fs) / (1024 * 1024))
else:
size = '%.2fGB' % (float(fs) / (1024 * 1024 * 1024))
keys = []
for k in self.ikeys(pattern):
keys.append(k)
keys.sort()
if keys: mlen = max([len(k) for k in keys]) + 2
else: mlen = 2
fmt = ' --> %%-%ds : %%s' % mlen
table = [('Key', 'type', ' size ', ' created/modified ')]
meta = self.get('__metainfo__', {})
for k in meta:
row = "META:%s" % k, '', '', str(meta[k])
table.append(row)
for k in keys:
size = ''
ss = self.__sizes.get(k, -1)
if ss < 0: size = ''
elif ss < 1024: size = '%7d ' % ss
elif ss < 1024 * 1024:
size = '%7.2f kB' % (float(ss) / 1024)
elif ss < 1024 * 1024 * 1024:
size = '%7.2f MB' % (float(ss) / (1024 * 1024))
else:
size = '%7.2f GB' % (float(ss) / (1024 * 1024 * 1024))
ot = type(self[k])
otype = ot.__cppname__ if hasattr(ot,
'__cppname__') else ot.__name__
rawitem = self.__get_raw_item__(k)
if isinstance(rawitem, Item): timetag = rawitem.time
else: timetag = ''
row = '{:15}'.format(k), '{:15}'.format(otype), size, timetag
table.append(row)
import ostap.logger.table as T
t = self.__class__.__name__
title = '%s:%s' % (t, n)
maxlen = 0
for row in table:
rowlen = 0
for i in row:
rowlen += len(i)
maxlen = max(maxlen, rowlen)
if maxlen + 3 <= len(title):
title = '<.>' + title[-maxlen:]
table = T.table(table, title=title, prefix='# ')
ll = getLogger(n)
line = 'Database %s:%s #keys: %d size: %s' % (t, ap, len(self), size)
ll.info('%s\n%s' % (line, table))
# @see https://github.com/uqfoundation/multiprocess
# @see https://github.com/uqfoundation/dill
# ============================================================================
""" Oversimplified script for parallel execution using multiprocessing
"""
# ============================================================================
import ROOT, time, sys
from itertools import count
from ostap.plotting.canvas import use_canvas
from ostap.utils.utils import wait
# =============================================================================
# logging
# =============================================================================
from ostap.logger.logger import getLogger
if '__main__' == __name__ or '__builtin__' == __name__ :
logger = getLogger ( 'test_parallel_multiprocess' )
else :
logger = getLogger ( __name__ )
# =============================================================================
try :
import dill
except ImportError :
logger.error('Can not import dill')
dill = None
# =============================================================================
try :
import multiprocess
except ImportError :
logger.error('Can not import multiprocess')
multiprocess = None
def test_breitwigner_rho():
"""Different rho0 parameterizations
"""
logger = getLogger("test_breitwigner_rho")
logger.info("Rho0 shapes")
## 1) P-wave Breit-Wigner with Jackson's formfactor
bw1 = Ostap.Math.Rho0(m_rho, g_rho, m_pi)
## 2) P-wave Breit-Wigner with Blatt-Weisskopf formfactor
ff = Ostap.Math.FormFactors.BlattWeisskopf(1, 3.5 / GeV)
ch2 = Ostap.Math.Channel(g_rho, m_pi, m_pi, 1, ff)
bw2 = Ostap.Math.BreitWigner(m_rho, ch2)
## 3) Gounaris-Sakurai lineshape
ch2 = Ostap.Math.ChannelGS(g_rho, m_pi)
bw3 = Ostap.Math.BreitWigner(m_rho, ch2)
## 3) P-wave Breit-Wigner with no formfactors
bw4 = Ostap.Math.BreitWigner(m_rho, g_rho, m_pi, m_pi, 1)
with use_canvas('test_breitwigner_rho'):
bw2.draw(xmin=200 * MeV, xmax=1.6 * GeV, linecolor=4)
bw1.draw('same', xmin=200 * MeV, xmax=1.6 * GeV, linecolor=2)
bw3.draw('same', xmin=200 * MeV, xmax=1.6 * GeV, linecolor=8)
bw4.draw('same', xmin=200 * MeV, xmax=1.6 * GeV, linecolor=5)
mass = ROOT.RooRealVar('mass', 'm(pipi)', 200 * MeV, 1.6 * GeV)
model1 = Models.BreitWigner_pdf('BW1',
bw1,
xvar=mass,
m0=m_rho,
gamma=g_rho)
model2 = Models.BreitWigner_pdf('BW2',
bw2,
xvar=model1.mass,
m0=model1.mean,
gamma=model1.gamma)
model3 = Models.BreitWigner_pdf('BW3',
bw3,
xvar=model1.mass,
m0=model1.mean,
gamma=model1.gamma)
with use_canvas('test_breitwigner_rho'):
f1 = model1.draw(total_fit_options=(ROOT.RooFit.LineColor(2), ))
f2 = model2.draw(total_fit_options=(ROOT.RooFit.LineColor(4), ))
f3 = model3.draw(total_fit_options=(ROOT.RooFit.LineColor(8), ))
f2.draw()
f1.draw('same')
f3.draw('same')
models.add(bw1)
models.add(bw2)
models.add(bw3)
models.add(model1)
models.add(model2)
models.add(model3)
time.sleep(2)