def initiaizeArgs(args):
global f
if args.file: # se scrittura su file
try:
f = open(args.file, 'w')
sys.stdout = f
except IOError:
print("error opening file: %s" % args.file)
if args.time:
cf.start_time = "'" + args.time + "'"
cf.prophet_diagnostic = args.prophet
if cf.prophet_diagnostic:
if args.dir:
if os.path.isdir(args.dir):
cf.graphDir = args.dir if args.dir[
-1] == '/' else args.dir + '/'
else:
print("directory not found: %s" % args.dir)
createDir(cf.graphDir)
else:
createDir(cf.graphDir)
cf.checkCat = args.cat
cf.showGraph = args.graph
cf.verbose = args.verbose
cf.mitigation = args.xdp
def packageDestinationDir(self) -> Path:
"""return absolute path to the directory where binary packages are placed into.
Default is to optionally append build type subdirectory"""
CraftCore.log.debug("CraftBase.packageDestinationDir called")
dstpath = Path(CraftCore.settings.get("Packager", "Destination", os.path.join(CraftStandardDirs.craftRoot(), "tmp")))
utils.createDir(dstpath)
return dstpath
def __init__(self, config): for key in config: setattr(self, key, config[key]) self.__setDefaultValues() self.__initBus() utils.createDir(self.module_path)
def testCustomerRelease(releaseType, releaseTarball, srcDir, binaryTarball=None, short=True, testlist=None, buildOnly=False, nprocs=1,parentTmpDir=None):
"""
Test a customer release by
binary release and vitamind release:
extract it
run tests
source release
extract binary release
extract source release
build source release and install into binary releaes
run tests
@param releaseType One of binaryReleaseTypes or sourceReleaseTypes
@param releaseTarball Absolute pathname to the tarball being tested
@param srcDir Absolute path to source tree (where tests may reside)
@param binaryTarball Absolute pathname to the binary tarball (used/required only
for the tools source release if buildOnly is False)
@param short If true, run short versions of hte tests
@param testlist Name of testlist. If None, then defaults to release type
@param buildOnly For source releases, just build -- don't run the tests.
@retval tuple of (pass, fail, disabled) tests
"""
# Require binary tarball if we're building a source release and
# either 1) we will test the release or 2) the release requires spnupic libraries to build
if releaseType in sourceReleaseTypes and binaryTarball is None:
if buildOnly == False or releaseType in releasesRequiringSPNupic:
raise Exception("testCustomerRelease: no binary tarball specified for release type %s" % releaseType)
if testlist is None:
testlist = releaseType
tmpDir = utils.createTemporaryDirectory("release_testing", parentDir=parentTmpDir)
if releaseType in binaryReleaseTypes:
tarballToExtract = releaseTarball
elif releaseType in sourceReleaseTypes:
tarballToExtract = binaryTarball
else:
raise Exception("testCustomerRelease: unknown release type '%s'" % releaseType)
if tarballToExtract is not None:
testdir = utils.extractArchive(tarballToExtract, tmpDir)
else:
testdir = os.path.join(tmpDir, "binary")
utils.createDir(testdir)
passed = list()
failed = list()
disabled = list()
log.info("Testing customer release %s in directory %s" % (releaseType, testdir))
if releaseType in sourceReleaseTypes:
try:
buildCustomerRelease(releaseType, srcDir, releaseTarball, binaryTarball, testdir, nprocs=nprocs)
except Exception, e:
log.error("Source release failed: %s", e)
failed = ["build source release"]
def getFile(url, destdir, filename='', quiet=None) -> bool:
"""download file from 'url' into 'destdir'"""
if quiet is None:
quiet = CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False)
CraftCore.log.debug("getFile called. url: %s" % url)
if url == "":
CraftCore.log.error("fetch: no url given")
return False
pUrl = urllib.parse.urlparse(url)
if not filename:
filename = os.path.basename(pUrl.path)
utils.createDir(destdir)
if pUrl.scheme == "s3":
return s3File(url, destdir, filename)
elif pUrl.scheme == "minio":
return minioGet(pUrl.netloc + pUrl.path, destdir, filename)
# curl and wget basically only work when we have a cert store on windows
if not CraftCore.compiler.isWindows or os.path.exists(os.path.join(CraftCore.standardDirs.etcDir(), "cacert.pem")):
if not CraftCore.settings.getboolean("General", "NoWget"):
if CraftCore.cache.findApplication("wget"):
return wgetFile(url, destdir, filename, quiet)
if CraftCore.cache.findApplication("curl"):
return curlFile(url, destdir, filename, quiet)
if os.path.exists(os.path.join(destdir, filename)):
return True
powershell = CraftCore.cache.findApplication("powershell")
if powershell:
filename = os.path.join(destdir, filename)
return utils.system([powershell, "-NoProfile", "-ExecutionPolicy", "ByPass", "-Command",
f"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; (new-object net.webclient).DownloadFile(\"{url}\", \"{filename}\")"])
else:
def dlProgress(count, blockSize, totalSize):
if totalSize != -1:
percent = int(count * blockSize * 100 / totalSize)
utils.printProgress(percent)
else:
sys.stdout.write(("\r%s bytes downloaded" % (count * blockSize)))
sys.stdout.flush()
try:
urllib.request.urlretrieve(url, filename=os.path.join(destdir, filename),
reporthook=dlProgress if CraftCore.debug.verbose() >= 0 else None)
except Exception as e:
CraftCore.log.warning(e)
return False
if CraftCore.debug.verbose() >= 0:
sys.stdout.write("\n")
sys.stdout.flush()
return True
def createCustomerRelease(releaseType, releaseSpecDir, destDir, fromDir, releaseName, allArchitectures=True, allArchiveTypes=False, allowSymbolicLinks=False, parentTmpDir=None):
"""Create a customer release tarball in destdir from the
directories engReleaseDir and releaseSpecDir
releaseType is one of biaryReleaseTypes or sourceReleaseTypes
fromDir is engineering build for the binary release,
source dir for a source release
releaseSpecDir is directory containing manifests
Returns a path to the tarball
Throws an exception on error"""
if not os.path.exists(destDir):
utils.createDir(destDir)
if releaseType in binaryReleaseTypes:
isSourceRelease = False
elif releaseType in sourceReleaseTypes:
isSourceRelease = True
else:
raise Exception("Unknown release type '%s'" % releaseType)
if not isSourceRelease:
allArchitectures=False
manifestFile = os.path.join(releaseSpecDir, releaseType + "_release",
"manifests", releaseType + "_release.manifest")
log.info("Creating customer release of type %s", releaseType)
log.info("specDir: %s", releaseSpecDir)
log.info("fromDir: %s", fromDir)
tempdir = utils.createTemporaryDirectory("assemble_release", parentDir=parentTmpDir)
try:
releaseImageDir = os.path.join(tempdir, releaseName)
installFromManifest(manifestFile, fromDir, releaseImageDir,
level=0, overwrite=False, destdirExists=False,
allArchitectures=allArchitectures, allowSymbolicLinks=allowSymbolicLinks)
if isSourceRelease:
# run autogen equivalent on source trees
build.prepareSourceTree(releaseImageDir, customerRelease=True)
# Need to create both zip and tgz source releases on the master build system
if allArchiveTypes == True:
# sanity checks
if not isSourceRelease:
raise Exception("Attempt to create both zip and tar file for a binary release. Probably an error")
if getArch() == "win32":
raise Exception("Attempt to create .tgz source release on a windows system. Would not include autogen.sh")
zipFile = utils.createArchive(releaseImageDir, destDir, type="zip")
tarFile = utils.createArchive(releaseImageDir, destDir, type="tar")
return (tarFile, zipFile)
else:
release = utils.createArchive(releaseImageDir, destDir)
return release
finally:
utils.remove(tempdir)
def install( self ):
if self.isTargetBuild():
# Configuring Qt for WinCE ignores the -prefix option,
# so we have to do the job manually...
# delete this because it is not working for windows
utils.deleteFile( os.path.join( self.buildDir(), "plugin", "bearer", "qnmbearerd4.dll" ))
utils.deleteFile( os.path.join( self.buildDir(), "plugin", "bearer", "qnmbearer4.dll" ))
# syncqt expects qconfig.h to be in the install dir and fails if not
utils.createDir( os.path.join( self.installDir(), "src", "corelib", "global") )
utils.copyFile( os.path.join( self.buildDir(), "src", "corelib", "global", "qconfig.h" ), os.path.join( self.installDir(), "src", "corelib" , "global", "qconfig.h" ) )
# headers need to be copied using syncqt because of the relative paths
utils.prependPath(self.sourceDir(), "bin")
command = os.path.join(self.sourceDir(), "bin", "syncqt.bat")
command += " -base-dir \"" + self.sourceDir() + "\""
command += " -outdir \"" + self.installDir() + "\""
command += " -copy"
command += " -quiet"
utils.system( command )
utils.copySrcDirToDestDir( os.path.join( self.buildDir(), "bin" ) , os.path.join( self.installDir(), "bin" ) )
utils.copySrcDirToDestDir( os.path.join( self.buildDir(), "lib" ) , os.path.join( self.installDir(), "lib" ) )
# the dlls must be copied to the bin dir too
for file in os.listdir( os.path.join( self.installDir(), "lib" ) ):
if file.endswith( ".dll" ):
utils.copyFile( os.path.join( self.installDir(), "lib" , file ), os.path.join( self.installDir(), "bin" , file ) )
utils.copySrcDirToDestDir( os.path.join( self.buildDir(), "mkspecs" ) , os.path.join( self.installDir(), "mkspecs" ) )
utils.copySrcDirToDestDir( os.path.join( self.buildDir(), "plugins" ) , os.path.join( self.installDir(), "plugins" ) )
# create qt.conf
utils.copyFile( os.path.join( self.packageDir(), "qt.conf" ), os.path.join( self.installDir(), "bin", "qt.conf" ) )
return True
if not QMakeBuildSystem.install(self):
return False
# Workaround QTBUG-12034
utils.copySrcDirToDestDir( os.path.join( self.buildDir(), "plugins", "imageformats" ) ,
os.path.join( self.installDir(), "bin", "imageformats" ) )
# create qt.conf
utils.copyFile( os.path.join( self.packageDir(), "qt.conf" ), os.path.join( self.installDir(), "bin", "qt.conf" ) )
# install msvc debug files if available
if self.buildType() == "Debug" and (self.compiler() == "msvc2005" or self.compiler() == "msvc2008" or self.compiler() == "msvc2010"):
srcdir = os.path.join( self.buildDir(), "lib" )
destdir = os.path.join( self.installDir(), "lib" )
filelist = os.listdir( srcdir )
for file in filelist:
if file.endswith( ".pdb" ):
utils.copyFile( os.path.join( srcdir, file ), os.path.join( destdir, file ) )
return True
def packageDestinationDir(self, withBuildType=True):
"""return absolute path to the directory where binary packages are placed into.
Default is to optionally append build type subdirectory"""
CraftCore.log.debug("CraftBase.packageDestinationDir called")
dstpath = CraftCore.settings.get(
"General", "EMERGE_PKGDSTDIR",
os.path.join(CraftStandardDirs.craftRoot(), "tmp"))
if not os.path.exists(dstpath):
utils.createDir(dstpath)
return dstpath
def packageDestinationDir( self, withBuildType=True ):
"""return absolute path to the directory where binary packages are placed into.
Default is to optionally append build type subdirectory"""
utils.debug( "EmergeBase.packageDestinationDir called", 2 )
dstpath = emergeSettings.get("General","EMERGE_PKGDSTDIR", os.path.join( EmergeStandardDirs.emergeRoot(), "tmp" ) )
if withBuildType:
if emergeSettings.getboolean("General", "EMERGE_MERGE_ROOT_WITH_BUILD_TYPE", False ):
dstpath = os.path.join( dstpath, self.buildType())
if not os.path.exists(dstpath):
utils.createDir(dstpath)
return dstpath
def qmerge( self ):
# When crosscompiling also install into the targets directory
ret = CMakePackageBase.qmerge(self)
if emergePlatform.isCrossCompilingEnabled():
target_mimedir = os.path.join(ROOTDIR, emergePlatform.targetPlatform(),
"share", "mime", "packages")
build_mime = os.path.join(self.imageDir(), "share", "mime",
"packages", "freedesktop.org.xml")
utils.createDir(target_mimedir)
if not os.path.exists(build_mime):
utils.error("Could not find mime file: %s" % build_mime)
return False
utils.copyFile(build_mime, target_mimedir)
return ret
def createEngineeringRelease(destDir, engReleaseDir, releaseName, tmpDir=None):
"""Create and engineering release tarball in destDir from the
directory engReleaseDir. Returns a path to the tarball
Throws an exception on error. """
log.info("Creating engineering release tarball")
if not os.path.exists(destDir):
utils.createDir(destDir)
engReleaseTarball = utils.createArchive(engReleaseDir,
destDir,
rootRename=releaseName,
tmpDir=tmpDir)
return engReleaseTarball
def createCab(self, cab_ini):
''' Create a cab file from cab_ini using Microsoft Cabwizard '''
cabwiz = os.path.join(os.getenv("TARGET_SDKDIR"), "..", "Tools",
"CabWiz", "Cabwiz.exe")
output = os.getenv("EMERGE_PKGDSTDIR", os.path.join(self.imageDir(),
"packages"))
if not os.path.exists(output):
utils.createDir(output)
elif not os.path.isdir(output):
utils.die("Outputpath %s can not be accessed." % output)
args = [cabwiz, cab_ini, "/dest", output, "/compress"]
utils.debug("Calling Microsoft Cabwizard: %s" % cabwiz, 2)
retcode = subprocess.call(args)
return retcode == 0
def packageDestinationDir( self, withBuildType=True ):
"""return absolute path to the directory where binary packages are placed into.
Default is to optionally append build type subdirectory"""
utils.debug( "EmergeBase.packageDestinationDir called", 2 )
dstpath = os.getenv( "EMERGE_PKGDSTDIR" )
if not dstpath:
dstpath = os.path.join( self.rootdir, "tmp" )
if withBuildType:
if utils.envAsBool( "EMERGE_MERGE_ROOT_WITH_BUILD_TYPE" ):
dstpath = os.path.join( dstpath, self.buildType())
if not os.path.exists(dstpath):
utils.createDir(dstpath)
return dstpath
def __init__(self, config):
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
self.nLoops = int(config.nLoops)
self.modelsDict = {
'AlexNet': models.alexnet,
'vgg16': models.vgg16,
'vgg19': models.vgg19,
}
self.nn = self.modelsDict[config.network]
self.nn = self.nn(pretrained=True).features.to(device=self.device)
self.logLoss = config.logLoss
self.saveSnapshotEvery = config.saveSnapshotEvery
self.dataLoader = dataLoader(config)
self.useWhiteNoise = config.startFromWhiteNoise
self.contentLayers = "".join(
config.contentLayers.split(" ")).split(',')
self.styleLayers = "".join(config.styleLayers.split(" ")).split(',')
self.alpha = config.alpha
self.beta = config.beta
self.poolingDict = {'Max': nn.MaxPool2d, 'Average': nn.AvgPool2d}
self.contentsLossModules = []
self.stylesLossModules = []
self.pooling = self.poolingDict[config.poolingStyle]
self.gram = lossModules.GramMatrix().to(device=self.device)
self.genIm = {}
for style in self.dataLoader.styleDataSet():
self.genIm[style] = {}
self.outputFolder = utils.createDir(config.outputFolder)
trans = [
transforms.Normalize(
mean=[-0.485 / 0.229, -0.456 / 0.224, -0.406 / 0.225],
std=[1 / 0.229, 1 / 0.224, 1 / 0.225])
]
trans.append(transforms.Lambda(lambda x: x.clamp_(0, 1)))
self.trans = transforms.Compose(trans)
self.keepColors = config.keepColors
def unpack( self ):
utils.cleanDirectory( self.buildDir() )
self.enterBuildDir()
print self.buildDir()
thirdparty_dir = os.path.join( self.buildDir(), "3rdparty" )
utils.createDir(thirdparty_dir)
if not utils.unpackFile( self.downloadDir(), os.path.basename(self.openssl), thirdparty_dir ):
return False
if not utils.unpackFile( self.downloadDir(), os.path.basename(self.dbuslib), thirdparty_dir ):
return False
if not utils.unpackFile( self.downloadDir(), os.path.basename(self.mysql), thirdparty_dir ):
return False
return True
def packageDestinationDir(self, withBuildType=True):
"""return absolute path to the directory where binary packages are placed into.
Default is to optionally append build type subdirectory"""
EmergeDebug.debug("EmergeBase.packageDestinationDir called", 2)
dstpath = emergeSettings.get(
"General", "EMERGE_PKGDSTDIR",
os.path.join(EmergeStandardDirs.emergeRoot(), "tmp"))
if withBuildType:
if emergeSettings.getboolean("General",
"EMERGE_MERGE_ROOT_WITH_BUILD_TYPE",
False):
dstpath = os.path.join(dstpath, self.buildType())
if not os.path.exists(dstpath):
utils.createDir(dstpath)
return dstpath
def _createShortPath(longPath) -> str:
if not CraftShortPath._useShortpaths:
return longPath
import utils
utils.createDir(CraftCore.standardDirs.junctionsDir())
longPath = OsUtils.toNativePath(longPath)
path = CraftCore.standardDirs.junctionsDir() / hex(
zlib.crc32(bytes(longPath, "UTF-8")))[2:]
delta = len(longPath) - len(str(path))
if delta <= 0:
CraftCore.debug.log.debug(
f"Using junctions for {longPath} wouldn't save characters returning original path"
)
CraftCore.debug.log.debug(f"{longPath}\n" f"{path}, gain: {delta}")
return longPath
utils.createDir(longPath)
if not os.path.isdir(path):
if OsUtils.isUnix():
ok = utils.createSymlink(longPath,
path,
useAbsolutePath=True,
targetIsDirectory=True)
else:
# note: mklink is a CMD command => needs shell
ok = utils.system(["mklink", "/J", path, longPath],
shell=True,
stdout=subprocess.DEVNULL,
logCommand=False)
if not ok:
CraftCore.debug.log.critical(
f"Could not create shortpath {path}, for {longPath}")
return longPath
else:
if not os.path.samefile(path, longPath):
CraftCore.debug.log.critical(
f"Existing short path {path}, did not match {longPath}")
return longPath
CraftCore.debug.log.debug(f"Mapped \n"
f"{longPath} to\n"
f"{path}, gained {delta}")
return path
def process(self, message):
self.message = message
end = message.get('end')
start = message.get('start')
tiles = message.get('tiles')
startDoy = message.get('startDoy')
startYear = message.get('startYear')
pathMrt = message.get('path_mrt')
product = message.get('product')
productName = message.get('productName')
layers = product['bands']
tiles = ",".join(tiles).lower()
noTiles = ('noTiles' in product and product['noTiles'])
if noTiles:
tiles = None
tmpDir = "_".join(['tmp', productName, startYear + startDoy])
tmpPath = os.path.join(self.module_path, tmpDir)
utils.createDir(tmpPath)
time.sleep(1)
self.__downloadAndReproject(productName=productName,
tiles=tiles,
start=start,
end=end,
layers=layers,
tmpPath=tmpPath,
pathMrt=pathMrt)
layerFiles = self.__moveLayerFiles(tmpPath, self.module_path)
#utils.removeDir(tmpPath)
for layer in self.__getLayers(layers, layerFiles):
message.set('layer', layer)
self.publish(message)
utils.log(self.name, 'Forward message (', productName,
layer['name'], ')')
def generate_hdf5(ftxt, output, fname, argument=False):
data = getDataFromTxt(ftxt)
F_imgs = []
F_landmarks = []
for (imgPath, landmarkGt, bbox) in data:
img = cv2.imread(imgPath, cv2.IMREAD_GRAYSCALE)
assert (img is not None)
logger("process %s" % imgPath)
# plt.imshow(img)
# plt.show()
f_face = img[int(bbox[0]):int(bbox[2]), int(bbox[1]):int(bbox[3])]
plt.imshow(f_face)
plt.show()
f_face = cv2.resize(f_face, (39, 39))
f_face = f_face.reshape((1, 39, 39))
f_landmark = landmarkGt.reshape((10))
F_imgs.append(f_face)
F_landmarks.append(f_landmark)
F_imgs, F_landmarks = np.asarray(F_imgs), np.asarray(F_landmarks)
F_imgs = processImage(F_imgs)
shuffle_in_unison_scary(F_imgs, F_landmarks)
# full face
base = join(OUTPUT, '1_F')
createDir(base)
output = join(base, fname)
logger("generate %s" % output)
with h5py.File(output, 'w') as h5:
h5['data'] = F_imgs.astype(np.float32)
h5['landmark'] = F_landmarks.astype(np.float32)
def run_experiment():
# load data
scores = list()
x_data, labels = load_files_object_size()
# one hot encode y
labels = tf.keras.utils.to_categorical(labels)
# shuffle dataset
x_data, labels = shuffle(x_data, labels)
# split data to train and test
trainX, test_X = x_data[:int(x_data.shape[0] *
0.9)], x_data[int(x_data.shape[0] * 0.9):]
trainy, test_y = labels[:int(x_data.shape[0] *
0.9)], labels[int(x_data.shape[0] * 0.9):]
# create dir of models if not existent
if not os.path.exists("models/"):
createDir("models/")
# train 10 different models
for r in range(10):
checkpoint_path = "models/training_" + str(r + 1) + "/cp.ckpt"
model = train_model(trainX, trainy, checkpoint_path)
test_X = test_X[:, :test_timesteps, :]
score = evaluate_model(model, test_X, test_y)
score = score * 100.0
print('>#%d: %.3f' % (r + 1, score))
scores.append(score)
# summarize results
summarize_results(scores)
def train_model(train_x, train_y, checkpoint_path=None):
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[
2], train_y.shape[1]
train_x = train_x[:, :train_timesteps, :]
# reshape data into time steps of sub-sequences
if conv:
n_steps = train_x.shape[1] / n_length
train_x = train_x.reshape(
(train_x.shape[0], int(n_steps), n_length, train_x.shape[2]))
# testX = test_x.reshape((test_x.shape[0], int(n_steps), n_length, trainX.shape[2]))
model = create_model(n_outputs, n_features, n_length)
# Display the model's architecture
if summary:
model.summary()
if os.path.exists(checkpoint_path + ".index"):
model.load_weights(checkpoint_path)
print("Previous Model Loaded")
else:
createDir(checkpoint_path)
# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
save_weights_only=True,
verbose=verbose)
# fit network
model.fit(train_x,
train_y,
epochs=epochs,
batch_size=batch_size,
verbose=verbose,
callbacks=[cp_callback],
validation_split=1 / fold)
return model
def process(self, message):
self.message = message
end = message.get('end')
start = message.get('start')
tiles = message.get('tiles')
startDoy = message.get('startDoy')
startYear = message.get('startYear')
pathMrt = message.get('path_mrt')
product = message.get('product');
productName = message.get('productName')
layers = product['bands'];
tiles = ",".join(tiles).lower()
noTiles = ('noTiles' in product and product['noTiles'])
if noTiles:
tiles = None
tmpDir = "_".join(['tmp', productName, startYear + startDoy])
tmpPath = os.path.join(self.module_path, tmpDir)
utils.createDir(tmpPath)
time.sleep(1)
self.__downloadAndReproject(productName=productName, tiles=tiles,
start=start, end=end, layers=layers,
tmpPath=tmpPath, pathMrt=pathMrt);
layerFiles = self.__moveLayerFiles(tmpPath, self.module_path)
#utils.removeDir(tmpPath)
for layer in self.__getLayers(layers, layerFiles):
message.set('layer', layer)
self.publish(message)
utils.log(self.name, 'Forward message (', productName , layer['name'], ')')
oneFile.Write()
twoFile.cd()
twoFile.Write()
testFile.cd()
testFile.Write()
print "Saved files: \n",testFile.GetName(), "\n", oneFile.GetName(), "\n", twoFile.GetName()
blah = []
u.drawAllInFile(oneFile, "MAD ele", twoFile, "MAD mu",None,"","", path, None,"norm", isLog=True)
#u.drawAllInFile(oneFile, "MCFM ele", twoFile, "Madgraph mu",None,"","", path, None,"norm")
#u.drawAllInFile(oneFile, "vbf ele", twoFile, "vbf mu",None,"","", path, None,"norm", isLog=True)
#u.drawAllInFile(twoFile, "MAD-125",None,"", None,"","", path, None,"norm")
u.createDir(path+"/eff")
h0 = oneFile.Get("gen_Mll_0")
h1 = oneFile.Get("gen_Mll_1")
h2 = oneFile.Get("gen_Mll_2")
r1 = h1.Clone()
r1.Divide(h0)
r2 = h2.Clone()
r2.Divide(h0)
r1.Draw("hist")
r2.Draw("hist same")
r1.SetMinimum(0)
r1.SetMaximum(1)
r1.SetTitle(";Mll gen at LHE; acc")
r1.SetLineColor(kRed+1)
def doInitialFits(subdir):
print '\t Loading up the files'
u.createDir(subdir)
basePath = cf.get("path","base")+'/batch_output/zgamma/8TeV/'
plotBase1 = cf.get("path","htmlbase")+'/html/zgamma/dalitz/fits-'+subdir+'/init'
print "\t ==== All the input files are initialized into these dictionaries ===="""
dataDict = {}
signalDict = {}
if hjp:
signalDict = {'hjp_mu2012_M125':TFile(basePath+cf.get("path","ver-jp")+'/jp-mugamma_2012/hhhh_HiggsToJPsiGamma_1.root','r')}
dataDict['mu2012'] = TFile(basePath+cf.get("path","ver-jp")+'/m_Data_jp-mugamma_2012.root','r')
else:
for y in yearList:
for l in leptonList:
if l == 'mu': tag = 'mugamma'
if l == 'el': tag = 'elgamma'
if l == 'ee': tag = 'eegamma'
if not os.path.exists(basePath):
print basePath, "does not exist!"
sys.exit(0)
if opt.tw and l=='el':
dataDict[l+y] = TFile('dalitzTW4/data.root','r')
else:
dataDict[l+y] = TFile(basePath+cf.get("path","ver-"+l)+'/m_Data_'+tag+'_'+y+'.root','r')
for s in sigNameList:
for m in massList:
if l=='el' and s=='v': continue
if l=='el' and opt.tw:
if s=='vbf':
signalDict[s+'_'+l+y+'_M'+m] = TFile('dalitzTW/output_job_hzg_eeg_dalitz_VBFH_'+m+'.root','r')
else:
signalDict[s+'_'+l+y+'_M'+m] = TFile('dalitzTW/output_job_hzg_eeg_dalitz_'+s+'H_'+m+'.root','r')
else:
signalDict[s+'_'+l+y+'_M'+m] = TFile(basePath+cf.get("path","ver-"+l)+'/'+tag+'_'+y+'/hhhh_'+s+'H-mad'+m+'_1.root','r')
print "\t === All files are set === \n"
apzTreeName = 'apzTree/apzTree'
twTreeName = 't'
binWidth = 2
binning = int(float(deltaMllg)/binWidth)
#binning = 30
weight = RooRealVar('Weight','Weight',0,100)
mzg = RooRealVar('CMS_hzg_mass','CMS_hzg_mass', lowCutOff,highCutOff)
mzg.setRange('FullRegion', lowCutOff, highCutOff)
mzg.setRange('DalitzRegion', lowCutOff, highCutOff)
mzg.setRange('SignalRegion', 120, 130)
mzg.setRange('myFitRange', 115, 135)
mzg.setBins(50000,'cache')
mzg.setRange('r1',110,120)
mzg.setRange('r2',130,170)
#res = RooRealVar('Mass_res','Mss_res', 0.8,1.2)
c = TCanvas("c","c",0,0,500,400)
c.cd()
ws = RooWorkspace("ws")
shist = u.AutoVivification()
yi_da0 = u.AutoVivification()
yi_da1 = u.AutoVivification()
yi_sig0 = u.AutoVivification()
yi_sig1 = u.AutoVivification()
mean_sig = u.AutoVivification()
sigma_sig = u.AutoVivification()
FWHM_sig = u.AutoVivification()
# ###################################
# start loop over all year/lep/cat #
# ###################################
for year in yearList:
for lepton in leptonList:
if hjp:
fs125 = TFile(subdir+'/s125-hjp.root','recreate')
else:
fs125 = TFile(subdir+'/s125-'+lepton+'.root','recreate')
if opt.tw and lepton=='el':
treeName = twTreeName
else:
treeName = apzTreeName
for cat in catList:
if lepton=='el' and cat!='EB': continue
plotBase = plotBase1+'_'.join([year,lepton,'cat'+cat])+'/'
u.createDir(plotBase)
if rootrace:
print "rootrace"
RooTrace.dump()
raw_input()
if verbose: print 'top of loop',year,lepton,cat
# ##################################################
# set up the signal histograms and the mzg ranges #
# ##################################################
for prod in sigNameList:
if lepton=='el' and prod=='v': continue
signalList = []
signalListDS = []
signalListDH = []
signalListPDF = []
for mass in massList:
histName = '_'.join(['sig',prod,lepton,year,'cat'+cat,'M'+mass])
shist[year][lepton][cat][prod][mass] = mzg.createHistogram(histName,
RooFit.Binning(binning))
# store the unbinned signals for CB fitting
signalTree = signalDict[prod+"_"+lepton+year+"_M"+mass].Get(treeName)
#signalTree.Print()
sigName = '_'.join(['ds_sig',prod,lepton,year,'cat'+cat,'M'+mass])
sig_argSW = RooArgSet(mzg,weight)
sig_ds = RooDataSet(sigName,sigName,sig_argSW,'Weight')
print signalTree, "A signal tree", prod, ' categ=',cat, "mass =", mass
if opt.tw and lepton=='el':
LoopOverTreeTW(signalTree, cat, mzg, sig_argSW, sig_ds)
else:
Nev = u.getTotalEvents(signalDict[prod+"_"+lepton+year+"_M"+mass])
sel = lepton
if sel=='ee': sel ='el'
lumiWeight = LumiXSWeighter(int(mass), prod, sel, Nev)
print 'Nev = ', Nev
LoopOverTree(signalTree, cat, mzg, sig_argSW, sig_ds, lumiWeight)
signalListDS.append(sig_ds)
getattr(ws,'import')(signalListDS[-1])
#raw_input()
sig_ds.fillHistogram(shist[year][lepton][cat][prod][mass], RooArgList(mzg))
if float(mass)==125:
fs125.cd()
shist[year][lepton][cat][prod][mass].Write()
signalList.append(shist[year][lepton][cat][prod][mass])
'''
signalTree = signalDict[prod+"_"+lepton+year+"_M"+mass].Get(treeName)
if verbose:
print 'signal mass loop', mass
print 'in gg INPUT ', cat, prod,mass
# raw_input()
print histName
signalTree.Print()
print
if cat=="0":
if noSFweight:
signalTree.Draw('m123>>'+histName,'')
else:
signalTree.Draw('m123>>'+histName,'weight')
elif cat=="EB":
signalTree.Draw('m123>>'+histName,"weight*(fabs(eta3)<"+str(EBetaCut)+")")
elif cat=="EE":
signalTree.Draw('m123>>'+histName,"weight*(fabs(eta3)>"+str(EEetaCut)+")")
if signalList[-1].Integral()!=0:
signalList[-1].Scale(1/signalList[-1].Integral())
'''
signalList[-1].Smooth(2)
# do some histogramming for gg signal for bias study
# range is +/- 1 RMS centered around signal peak
rangeLow = signalList[-1].GetMean()-1.0*signalList[-1].GetRMS()
rangeHi = signalList[-1].GetMean()+1.0*signalList[-1].GetRMS()
rangeName = '_'.join(['range',lepton,year,'cat'+cat,'M'+mass])
mzg.setRange(rangeName,rangeLow,rangeHi)
mzg_argL = RooArgList(mzg)
mzg_argS = RooArgSet(mzg)
signalListDH.append(RooDataHist('dh_'+histName, 'dh_' +histName,mzg_argL,signalList[-1]))
signalListPDF.append(RooHistPdf('pdf_'+histName,'pdf_'+histName,mzg_argS,signalListDH[-1],2))
getattr(ws,'import')(signalListPDF[-1])
if verbose: print '\n\n\t ** Finished the mass -->>', mass, '\n\n'
yi_sig0[year][mass][lepton][cat][prod] = sig_ds.sumEntries()
yi_sig1[year][mass][lepton][cat][prod] = sig_ds.sumEntries('1','SignalRegion')
reduced = signalListDS[-1].reduce("CMS_hzg_mass>"+str(float(mass)*(1-0.1))+"&&CMS_hzg_mass<"+str(float(mass)*(1+0.1)))
mean_sig[year][mass][lepton][cat][prod] = [reduced.mean(mzg), signalList[-1].GetMeanError()]
sigma_sig[year][mass][lepton][cat][prod] = [reduced.sigma(mzg), signalList[-1].GetRMSError()]
#mean_sig[year][mass][lepton][cat][prod] = [signalListDS[-1].mean(mzg), signalList[-1].GetMeanError()]
#sigma_sig[year][mass][lepton][cat][prod] = [signalListDS[-1].sigma(mzg), signalList[-1].GetRMSError()]
#mean_sig[year][mass][lepton][cat][prod] = [signalList[-1].GetMean(), signalList[-1].GetMeanError()]
#sigma_sig[year][mass][lepton][cat][prod] = [signalList[-1].GetRMS(), signalList[-1].GetRMSError()]
bin1 = signalList[-1].FindFirstBinAbove(signalList[-1].GetMaximum()/2)
bin2 = signalList[-1].FindLastBinAbove(signalList[-1].GetMaximum()/2)
print bin1, bin2
FWHM_sig[year][mass][lepton][cat][prod] = (signalList[-1].GetBinCenter(bin2) - signalList[-1].GetBinCenter(bin1))
# mean_dh[year][mass][lepton][cat] = [signalListDH[-1].GetMean(), signalListDH[-1].GetMeanError()]
# sigma_dh[year][mass][lepton][cat] = [signalListDH[-1].GetRMS(), signalListDH[-1].GetRMSError()]
if debugPlots:
fitBuilder = FitBuilder(mzg, year, lepton, cat)
print '\n\n Now make some plots!\n'
testFrame = mzg.frame()
SigFits = {'0':None} # Format 'mass': fit-function
for i,signal in enumerate(signalListPDF):
signalListDH[i].plotOn(testFrame)
signal.plotOn(testFrame, RooFit.Name(sigName))
if lepton=='mu': testFrame.SetTitle(';m_{#mu#mu#gamma} (GeV);Events')
else: testFrame.SetTitle(';m_{ee#gamma} (GeV);Events')
testFrame.SetMinimum(0.0001)
testFrame.Draw()
CMS_lumi(c, 2, 11, 'Simulation')
c.SaveAs(plotBase+'_'.join(['signals',prod,year,lepton,'cat'+cat])+'.png')
# testFrame = mzg.frame()
for signal in signalListDS:
"""
sigName = '_'.join(['DBG-DH',prod,lepton,year,'cat'+cat,'M'+mass])
SigFits[sigName],paramList = fitBuilder.Build('Gauss', mean = 125, meanLow = 120, meanHigh = 130, sigma = 0.2, sigmaLow = 0.05, sigmaHigh = 1)
SigFits[sigName].fitTo(signalListDS[i], RooFit.Range('myFitRange'), RooFit.PrintLevel(-1))
print paramList
for param in paramList:
print sigName, param.GetName(), param.getVal()
raw_input("\n ---> Param List. hit enter to continue")
SigFits[sigName].plotOn(testFrame)
SigFits[sigName].paramOn(testFrame)
"""
testFrame = mzg.frame()
signal.plotOn(testFrame, RooFit.DrawOption('pl'))
testFrame.Draw()
myS125 = shist[year][lepton][cat][prod]['125'].Clone()
myS125.Draw('same hist')
myS125.SetLineColor(kGreen+1)
f1 = TF1("f1", "gaus", 120, 130);
f1.SetParameters(0,125, 0.2);
f1.FixParameter(0, 0);
f1.SetParLimits(1, 122, 127);
myS125.Fit('f1','R')
# myS125.GetFunction("f1").Print()
#raw_input("\n ---> Param List. hit enter to continue")
CMS_lumi(c, 2, 11, 'Simulation')
c.SaveAs(plotBase+'_'.join(['ds_sig',prod,year,lepton,'cat'+cat])+'.png')
del signalTree
# ###############
# get the data #
# ###############
if verbose: print '\n\n \t ** Starting data selection. \n'
dataName = '_'.join(['data',lepton,year,'cat'+cat])
dataTree = dataDict[lepton+year].Get(treeName)
data_argS = RooArgSet(mzg)
data_argL = RooArgList(mzg)
data_ds = RooDataSet(dataName,dataName,data_argS)
if opt.tw and lepton=='el':
LoopOverTreeTW(dataTree, cat, mzg, data_argS, data_ds)
else:
LoopOverTree(dataTree, cat, mzg, data_argS, data_ds, None)
if verbose: data_ds.Print('all')
dahist = mzg.createHistogram('hist_'+dataName, RooFit.Binning(binning))
data_ds.fillHistogram(dahist, RooArgList(mzg))
data_dh = RooDataHist('dh_'+dataName, 'dh_' +dataName, data_argL, dahist)
#dahist.Print('all')
# data_dh.Print()
if verbose:
print dataName
data_ds.Print()
print
if debugPlots:
testFrame = mzg.frame()
data_ds.plotOn(testFrame,RooFit.Binning(binning))
testFrame.Draw()
CMS_lumi(c, 2, 11)
c.SaveAs(plotBase+'_'.join(['data',year,lepton,'cat'+cat])+'.png')
getattr(ws,'import')(data_ds)
# ############
# make fits #
# ############
if verbose: '\t\t *** Starting BKG fits ***\n'
fitBuilder = FitBuilder(mzg, year, lepton, cat)
bgFitList = ['Exp','Pow','Bern2','Bern3','Bern4','Bern5','Laurent']
testFrame = mzg.frame(RooFit.Range('DalitzRegion'))
#if doBlind:
# data_ds.plotOn(testFrame, RooFit.Binning(binning),RooFit.Name('r1'),RooFit.CutRange('r1'))
# data_ds.plotOn(testFrame, RooFit.Binning(binning),RooFit.Name('r2'),RooFit.CutRange('r2'))
# data_ds.plotOn(testFrame, RooFit.Binning(binning),RooFit.Name('data'),RooFit.Invisible())
#else:
# data_ds.plotOn(testFrame, RooFit.Binning(binning),RooFit.Name('data'))
data_dh.plotOn(testFrame, RooFit.Binning(binning), RooFit.Name('data2'))
leg = TLegend(0.63,0.6,0.91,0.88)
leg.SetFillColor(0)
leg.SetShadowColor(0)
leg.SetBorderSize(1)
leg.SetHeader(', '.join([year,lepton,'cat: '+cat]))
for fitName in bgFitList:
ndof = fitBuilder.FitNdofDict[fitName]
fit = fitBuilder.Build(fitName)
if verbose:
fit.Print()
fit.fitTo(data_dh,RooFit.Range('DalitzRegion'), RooFit.Strategy(1))
#fit.fitTo(data_ds,RooFit.Range('DalitzRegion'), RooFit.Strategy(1))
fit.plotOn(testFrame, RooFit.LineColor(colors[fitName.lower()]), RooFit.Name(fitName))
getattr(ws,'import')(fit)
testFrame.Draw()
chi2 = testFrame.chiSquare(fitName,'data2',ndof)
leg.AddEntry(testFrame.findObject(fitName),fitName+(10-len(fitName))*' '+'#chi2 = {0:.2f}'.format(chi2),'l')
if doBlind:
testFrame.SetMinimum(0.1)
if lepton=='mu':
testFrame.SetTitle(";m_{#mu#mu#gamma} (GeV);Events/"+str(binWidth)+" GeV")
else:
testFrame.SetTitle(";m_{ee#gamma} (GeV);Events/"+str(binWidth)+" GeV")
testFrame.Draw()
# dahist.Draw('same hist')
#shist[year][lepton][cat]['gg']['125'].Scale(10)
#shist[year][lepton][cat]['gg']['125'].Draw('same hist')
#shist[year][lepton][cat]['gg']['125'].SetLineColor(kRed+1)
leg.Draw()
CMS_lumi(c, 2, 11)
c.SaveAs(plotBase+'_'.join(['fits',year,lepton,'cat'+cat])+'.png')
ws.commitTransaction()
print '\t Commit transaction end \n'
yi_da0[year][lepton][cat] = data_ds.sumEntries()
yi_da1[year][lepton][cat] = data_ds.sumEntries('1','SignalRegion')
fs125.Close()
ws.writeToFile(subdir+'/testRooFitOut_Dalitz.root')
print '*** Some yields from data'
print 'Full range:', yi_da0
if not doBlind:
print 'in SignalRegion:', yi_da1
if verbose:
ws.Print()
print '*** Some yields from the Signal'
print 'Full range:', yi_sig0
print 'in SignalRegion:', yi_sig1
print "\n ** You should also notice that EBeta cut was: ", EBetaCut
print "And that the range was from ", lowCutOff, 'to', highCutOff
print "\n Mean from dataset: ", mean_sig
print "\n Sigma DS:", sigma_sig
print "\n FWHM:", FWHM_sig
# print "\n Mean from Datahist:", mean_gg0_dh
# print "\n Sigma DH:", sigma_gg0_dh
raw_input('Enter')
if verbose:
for year in yearList:
for lepton in leptonList:
for prod in sigNameList:
if lepton=='el' and prod=='v': continue
for cat in catList:
t_mean = []
t_sigma = []
t_fwhm = []
for mass in massList:
l_mean = []
l_sigma = []
l_fwhm = []
l_mean.append(mass)
l_sigma.append(mass)
l_fwhm.append(mass)
print year, lepton, cat, prod
print mean_sig[year][mass][lepton][cat][prod]
print sigma_sig[year][mass][lepton][cat][prod]
print FWHM_sig[year][mass][lepton][cat][prod]
# a,b = mean_sig[year][mass][lepton][cat][prod]
# l.append("%.2f ± %.2f"%(a,b))
l_mean.append("%.3f±%.3f" %(mean_sig[year][mass][lepton][cat][prod][0], mean_sig[year][mass][lepton][cat][prod][1]))
l_sigma.append("%.3f±%.3f"%(sigma_sig[year][mass][lepton][cat][prod][0],sigma_sig[year][mass][lepton][cat][prod][1]))
l_fwhm.append("%.3f"% (FWHM_sig[year][mass][lepton][cat][prod]))
t_mean.append(l_mean)
t_sigma.append(l_sigma)
t_fwhm.append(l_fwhm)
u.makeTable(t_mean, "mean", opt="twiki")
u.makeTable(t_sigma, "sigma", opt="twiki")
u.makeTable(t_fwhm, "fwhm", opt="twiki")
print '\n \t We did it!\t'
sys.path.append("../zgamma")
import utils as u
import ConfigParser as cp
cf = cp.ConfigParser()
cf.read('config.cfg')
gROOT.ProcessLine(".L ~/tdrstyle.C")
setTDRStyle()
gStyle.SetOptTitle(0)
s = cf.get("fits","ver")
out = TFile(s+"/limit-data.root","recreate")
plotBase = '/uscms_data/d2/andreypz/html/zgamma/dalitz/fits-'+s
u.createDir(plotBase)
fullCombo = True
byParts = False
massList = [float(a.strip()) for a in (cf.get("fits","massList")).split(',')]
c = TCanvas("c","c",0,0,500,400)
c.cd()
if fullCombo:
xAxis = []
obs = []
exp = []
exp1SigHi = []
exp1SigLow = []
doMerge = options.merge
period = options.period
doBkg = options.bkg
gROOT.ProcessLine(".L ../tdrstyle.C")
setTDRStyle()
TH1.SetDefaultSumw2(kTRUE)
pathBase = "/uscms_data/d2/andreypz/html/zgamma/dalitz/"+ver+"_cut"+cut
hPath = "/eos/uscms/store/user/andreypz/batch_output/zgamma/8TeV/"+ver
if '/tthome' in os.getcwd():
pathBase = "/tthome/andrey/html/zgamma/dalitz/"+ver+"_cut"+cut
hPath = "/tthome/andrey/batch_output/zgamma/8TeV/"+ver
u.createDir(pathBase)
if doMerge:
os.system("rm "+hPath+"/m_*.root") #removing the old merged files
dataFile = None
bkgFiles = []
bkgNames = []
u.setSelection(sel)
if doMerge:
os.system("hadd "+hPath+"/m_Data_" +sel+"_"+period+".root "+hPath+"/"+sel+"_"+period+"/hhhh_DoubleMu_Run20*.root")
# os.system("hadd "+hPath+"/m_Data_" +sel+"_"+period+".root "+hPath+"/"+sel+"_"+period+"/hhhh_MuEG_Run20*.root")
# os.system("hadd "+hPath+"/m_Data_" +sel+"_"+period+".root "+hPath+"/"+sel+"_"+period+"/hhhh_DoubleElectron_Run20*.root")
def buildit(rootdir, srcdir, builddir, installdir, svnpath, stamp, revision, datestamp, assertions, update, nprocs=1):
"""
Do a full checkout and build
"""
arch = getArch()
log.info("Building for architecture " + arch)
#
# If source dir already exists, we're doing an incremental checkout. If not, we're
# doing a full checkout.
#
incremental = True
if not os.path.exists(srcdir):
if not update:
log.error("No svn update requested, but source dir '%s' does not exist.", srcdir)
raise Exception()
else:
incremental = False
log.info("Source directory %s does not exist. Will do a full checkout", srcdir)
else:
if not os.path.isdir(srcdir):
log.error("Source directory %s is not a directory.", srcdir)
raise Exception()
log.debug("Creating release directory %s", installdir)
utils.createDir(installdir, True)
utils.changeDir(os.path.join(installdir, os.pardir))
buildfilename = os.path.join(installdir, ".buildinfo")
#
# Bring source tree up to date if requested
#
if update:
log.info("Bringing source tree up to date at svnpath %s revision %s", svnpath, revision)
svn.checkout(srcdir, repo=svnpath, revision=revision, incremental=incremental, keepLocalMods=False)
else:
log.warn("Skipping bringing up to date. Build will be tainted")
utils.touchFile(os.path.join(installdir, "TAINTED_BUILD"))
#
# Revision may not be a number. May be "HEAD" or "PREV" or something else, so
# retrieve it from svn
#
revisionnum = svn.getRevision(srcdir)
#
# place a temporary file in the install dir to indicate a build in progress
# the file is removed at the end of the build process
#
build_in_progress_filename = os.path.join(installdir,"UNFINISHED_BUILD." + arch)
utils.touchFile(build_in_progress_filename)
log.debug("Updating build description file " + buildfilename)
try:
f = open(buildfilename, mode = "a")
print >> f, "============================="
print >> f, "Timestamp: " + datestamp
print >> f, "Arch: " + arch
print >> f, "Buildhost: " + platform.node()
print >> f, "Svnpath: " + svnpath
print >> f, "Revision: " + str(revisionnum)
print >> f, "Rootdir: " + rootdir
print >> f, "Srcdir: " + srcdir
print >> f, "Stamp: " + stamp
print >> f, "Assertions:" + str(assertions)
print >> f, "Update: " + str(update)
print >> f, ""
f.close()
except Exception, e:
raise Exception("Error writing to build description file '%s'", buildfilename)
def getFile(url, destdir, filename='', quiet=None) -> bool:
"""download file from 'url' into 'destdir'"""
if quiet is None:
quiet = CraftCore.settings.getboolean("ContinuousIntegration",
"Enabled", False)
CraftCore.log.debug("getFile called. url: %s" % url)
if url == "":
CraftCore.log.error("fetch: no url given")
return False
pUrl = urllib.parse.urlparse(url)
if not filename:
filename = os.path.basename(pUrl.path)
utils.createDir(destdir)
if pUrl.scheme == "s3":
return s3File(url, destdir, filename)
elif pUrl.scheme == "minio":
return minioGet(pUrl.netloc + pUrl.path, destdir, filename)
absFilename = Path(destdir) / filename
# try the other methods as fallback if we are bootstrapping
bootStrapping = not (CraftCore.standardDirs.etcDir() /
"cacert.pem").exists()
if not CraftCore.settings.getboolean("General", "NoWget"):
if CraftCore.cache.findApplication("wget"):
if wgetFile(url, destdir, filename, quiet):
return True
if not bootStrapping:
return False
if CraftCore.cache.findApplication("curl"):
if curlFile(url, destdir, filename, quiet):
return True
if not bootStrapping:
return False
if bootStrapping and absFilename.exists():
os.remove(absFilename)
if absFilename.exists():
return True
CraftCore.log.info(f"Downloading: {url} to {absFilename}")
with utils.ProgressBar() as progress:
def dlProgress(count, blockSize, totalSize):
if totalSize != -1:
progress.print(int(count * blockSize * 100 / totalSize))
else:
sys.stdout.write(
("\r%s bytes downloaded" % (count * blockSize)))
sys.stdout.flush()
try:
urllib.request.urlretrieve(
url,
filename=absFilename,
reporthook=dlProgress
if CraftCore.debug.verbose() >= 0 else None)
except Exception as e:
CraftCore.log.warning(e)
powershell = Powershell()
if powershell.pwsh:
return powershell.execute([
f"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; (new-object net.webclient).DownloadFile(\"{url}\", \"{absFilename}\")"
])
return False
return True
def copy_files(self):
'''
Copy the binaries for the Package from kderoot to the image
directory
'''
wm_root = os.path.join(self.rootdir,
os.environ["EMERGE_TARGET_PLATFORM"]) + os.path.sep
utils.createDir(self.workDir())
utils.debug("Copying from %s to %s ..." % (wm_root,
self.workDir()), 1)
uniquebasenames = []
unique_names = []
duplicates = []
#FIXME do this in the gpg-wce-dev portage
windir = os.path.join(wm_root, "windows")
if not os.path.isdir(windir):
os.mkdir(windir)
libgcc_src = os.path.join(wm_root, "bin", "libgcc_s_sjlj-1.dll")
libgcc_tgt = os.path.join(wm_root, "windows", "libgcc_s_sjlj-1.dll")
utils.copyFile(libgcc_src, libgcc_tgt)
for entry in self.traverse(wm_root, self.whitelisted):
if os.path.basename(entry) in uniquebasenames:
utils.debug("Found duplicate filename: %s" % \
os.path.basename(entry), 2)
duplicates.append(entry)
else:
unique_names.append(entry)
uniquebasenames.append(os.path.basename(entry))
for entry in unique_names:
entry_target = entry.replace(wm_root,
os.path.join(self.workDir()+"\\"))
if not os.path.exists(os.path.dirname(entry_target)):
utils.createDir(os.path.dirname(entry_target))
shutil.copy(entry, entry_target)
utils.debug("Copied %s to %s" % (entry, entry_target),2)
dups=0
for entry in duplicates:
entry_target = entry.replace(wm_root,
os.path.join(self.workDir()+"\\"))
entry_target = "%s_dup%d" % (entry_target, dups)
dups += 1
if not os.path.exists(os.path.dirname(entry_target)):
utils.createDir(os.path.dirname(entry_target))
shutil.copy(entry, entry_target)
utils.debug("Copied %s to %s" % (entry, entry_target), 2)
# Handle special cases
pinentry = os.path.join(wm_root, "bin", "pinentry-qt.exe")
utils.copyFile(pinentry, os.path.join(self.workDir(), "bin",
"pinentry.exe"))
#The Kolab icon is in hicolor but we only package oxygen for CE
kolabicon = os.path.join(wm_root, "share", "icons", "hicolor",
"64x64", "apps", "kolab.png")
utils.copyFile(kolabicon, os.path.join(self.workDir(), "share",
"icons", "oxygen", "64x64", "apps", "kolab.png"))
# Drivers need to be installed in the Windows direcotry
gpgcedev = os.path.join(wm_root, "bin", "gpgcedev.dll")
if not os.path.isdir(os.path.join(self.workDir(), "windows")):
os.mkdir(os.path.join(self.workDir(), "windows"))
utils.copyFile(gpgcedev, os.path.join(self.workDir(), "windows"))
# Create an empty file for DBus and his /etc/dbus-1/session.d
dbus_session_d = os.path.join(self.workDir(), "etc", "dbus-1",
"session.d")
if not os.path.exists(dbus_session_d): os.mkdir(dbus_session_d)
f = open(os.path.join(dbus_session_d, "stub"), "w")
f.close()
# Rename applications that should be started by the loader
for f in self.loader_executables:
path2file = os.path.join(wm_root, f)
if not os.path.isfile(path2file):
utils.debug("Createloaderfiles: Could not find %s at %s " % \
(f, path2file))
continue
realpath = path2file.replace(f, os.path.splitext(f)[0] +
"-real.exe")
shutil.copy(path2file, realpath.replace(wm_root,
os.path.join(self.workDir()+"\\")))
customloader = path2file.replace(f, os.path.splitext(f)[0] +
"-loader.exe")
defaultloader = os.path.join(wm_root, "bin", "himemce.exe")
path2file = path2file.replace(wm_root,
os.path.join(self.workDir()+"\\"))
if os.path.isfile(customloader):
shutil.copy(customloader, path2file)
elif os.path.isfile(defaultloader):
shutil.copy(defaultloader, path2file)
else:
utils.die("Trying to use the custom loader but no loader \
found in: %s \n Please ensure that package wincetools is installed" %\
os.path.join(wm_root, "bin"))
# Add start menu icons to the Package
for f in self.menu_icons:
path2file = os.path.join(self.packageDir(), "icons", f)
if os.path.isfile(path2file):
utils.copyFile(path2file, os.path.join(self.workDir(),
"share", "icons"))
else:
utils.debug("Failed to copy %s not a file: %s" % path2file, 1)
# Configure localization
if self.buildTarget == 'de':
confdir = os.path.join(self.workDir(), "share", "config")
if not os.path.isdir(confdir):
os.makedirs(confdir)
with open(os.path.join(confdir, "kdeglobals"),"w") as f:
f.write('[Locale]\n')
f.write('Country=de\n')
f.write('Language=de\n\n')
f.write('[Spelling]\n')
f.write('defaultLanguage=de_DE\n')
# Configure GNUPG
confdir = os.path.join(self.workDir(), "gnupg")
if not os.path.isdir(confdir):
os.makedirs(confdir)
if os.getenv("EMERGE_PACKAGE_CUSTOM_LDAPSERVERS_CONF"):
utils.copyFile(os.getenv("EMERGE_PACKAGE_CUSTOM_LDAPSERVERS_CONF"),
os.path.join(confdir, "ldapservers.conf"))
else:
utils.copyFile(os.path.join(self.packageDir(), "ldapservers-example.conf"),
os.path.join(confdir, "ldapservers.conf"))
# Add trustlist
# if os.getenv("EMERGE_PACKAGE_CUSTOM_TRUSTLIST_TXT"):
# utils.copyFile(os.getenv("EMERGE_PACKAGE_CUSTOM_TRUSTLIST_TXT"),
# os.path.join(confdir, "trustlist.txt"))
# else:
# utils.copyFile(os.path.join(self.packageDir(), "trustlist-example.txt"),
# os.path.join(confdir, "trustlist.txt"))
# Add certificates
if os.getenv("EMERGE_PACKAGE_CUSTOM_ROOTCERTS"):
confdir = os.path.join(self.workDir(), "gnupg", "trusted-certs")
cpath = os.getenv("EMERGE_PACKAGE_CUSTOM_ROOTCERTS")
if not os.path.isdir(confdir):
os.mkdir(confdir)
if not os.path.isdir(cpath):
utils.die("EMERGE_PACKAGE_CUSTOM_ROOTCERTS set up invalid")
for f in os.listdir(cpath):
utils.copyFile(os.path.join(cpath, f), confdir)
# Configure Strigi
confdir = os.path.join(self.workDir(), "My Documents", ".strigi")
if not os.path.isdir(confdir):
os.makedirs(confdir)
utils.copyFile(os.path.join(self.packageDir(), "daemon.conf"),
os.path.join(confdir, "daemon.conf"))
# Locale information
confdir = os.path.join(self.workDir(), "share", "locale", "l10n", "de")
if not os.path.isdir(confdir):
os.makedirs(confdir)
# hack for duplicate file, there cannot be over 21000 files in the cab
# so should be save
utils.copyFile(os.path.join(self.packageDir(), "entry.desktop"),
os.path.join(confdir, "entry.desktop_dup99999999"))
# Configure Kleopatra
confdir = os.path.join(self.workDir(), "My Documents", ".kde",
"share", "config")
if not os.path.isdir(confdir):
os.makedirs(confdir)
with open(os.path.join(confdir, "kleopatrarc"),"w") as f:
f.write("[Self-Test]\n")
f.write("run-at-startup=false\n\n")
f.write("[CertificateCreationWizard]\n")
f.write("DNAttributeOrder=CN!,EMAIL!\n")
def SignalFitMaker(lep, year, cat, subdir):
# reasd these from a config file:
massList = [a.strip() for a in (cf.get("fits","massList")).split(',')]
sigNameList = [a.strip() for a in (cf.get("fits","sigNameList")).split(',')]
plotBase = "/uscms_data/d2/andreypz/html/zgamma/dalitz/fits-"+subdir
u.createDir(plotBase)
rooWsFile = TFile(subdir+'/testRooFitOut_Dalitz.root')
myWs = rooWsFile.Get('ws')
#myWs.Print()
RooRandom.randomGenerator().SetSeed(8675309)
c = TCanvas("c","c",0,0,500,400)
c.cd()
mzg = myWs.var('CMS_hzg_mass')
cardDict = AutoVivification()
for mass in massList:
cardDict[lep][year][cat][mass] = RooWorkspace('ws_card')
print cardDict[lep][year][cat][mass]
# we need crystal ball + gaussian fits for all mass points, and for all production methods.
# we also need to interpolate the distributions for 0.5 mass bins, so we use some tricks
# in order to create new fits out of the existing 5GeV steps
# ##################
# Start the Loop! #
# ##################
for prod in sigNameList:
dsList = []
fitList = []
normList = []
oldMassHi = oldMassLow = 0
for massString in massList:
# ############################################
# Get the high and low mass references #
# If the point does not need interpolation, #
# we just use it for high and low #
# ############################################
mass = float(massString)
if mass%5.0 == 0.0:
massHi = int(mass)
massLow = int(mass)
else:
massRound = roundTo5(massString)
if mass<massRound:
massHi = massRound
massLow = massRound-5
else:
massHi = massRound+5
massLow = massRound
###### only calc the low and high points if they change
if not(oldMassLow == massLow and oldMassHi == massHi):
oldMassLow = massLow
oldMassHi = massHi
###### fit the low mass point
#if massLow<=125:
if massLow<=100:
mzg.setRange('fitRegion1',115,int(massLow)+10)
else:
mzg.setRange('fitRegion1',int(massLow)-15,int(massLow)+10)
sigNameLow = '_'.join(['ds','sig',prod,lep,year,'cat'+cat,'M'+str(massLow)])
sig_ds_Low = myWs.data(sigNameLow)
if massLow == massHi: dsList.append(sig_ds_Low)
CBG_Low = BuildCrystalBallGauss(year,lep,cat,prod,str(massLow),'Low',mzg,mean = massLow)[0]
CBG_Low.fitTo(sig_ds_Low, RooFit.Range('fitRegion1'), RooFit.SumW2Error(kTRUE), RooFit.Strategy(1), RooFit.NumCPU(4), RooFit.PrintLevel(-1))
###### fit the hi mass point
#if massHi<=125:
if massHi<=100:
mzg.setRange('fitRegion2',115,int(massHi)+10)
else:
mzg.setRange('fitRegion2',int(massHi)-15,int(massHi)+10)
sigNameHi = '_'.join(['ds','sig',prod,lep,year,'cat'+cat,'M'+str(massHi)])
sig_ds_Hi = myWs.data(sigNameHi)
CBG_Hi = BuildCrystalBallGauss(year,lep,cat,prod,str(massHi),'Hi',mzg,mean = massHi)[0]
CBG_Hi.fitTo(sig_ds_Hi, RooFit.Range('fitRegion2'), RooFit.SumW2Error(kTRUE), RooFit.Strategy(1), RooFit.NumCPU(4), RooFit.PrintLevel(-1))
###### interpolate the two mass points
massDiff = (massHi - mass)/5.
#if mass<=125:
if mass<=100:
mzg.setRange('fitRegion_'+massString,115,mass+10)
else:
mzg.setRange('fitRegion_'+massString,mass-15,mass+15)
beta = RooRealVar('beta','beta', 0.5, 0., 1.)
if massHi == massLow:
beta.setVal(1);
else:
beta.setVal(massDiff)
interp_pdf = RooIntegralMorph('interp_pdf', 'interp_pdf', CBG_Low, CBG_Hi, mzg, beta)
interp_ds = interp_pdf.generate(RooArgSet(mzg), 10000)
normList.append(sig_ds_Low.sumEntries()*massDiff+sig_ds_Hi.sumEntries()*(1-massDiff))
yieldName = '_'.join(['sig',prod,'yield',lep,year,'cat'+cat])
yieldVar = RooRealVar(yieldName,yieldName,sig_ds_Low.sumEntries()*massDiff+sig_ds_Hi.sumEntries()*(1-massDiff))
sigNameInterp = '_'.join(['ds','sig',prod,lep,year,'cat'+cat,'M'+str(mass)])
CBG_Interp,paramList = BuildCrystalBallGauss(year,lep,cat,prod,str(mass),'Interp',mzg,mean = mass)
CBG_Interp.fitTo(interp_ds, RooFit.Range('fitRegion_'+massString), RooFit.SumW2Error(kTRUE), RooFit.Strategy(1), RooFit.NumCPU(4), RooFit.PrintLevel(-1))
for param in paramList:
param.setConstant(True)
fitList.append(CBG_Interp)
print lep,year,cat,mass, cardDict[lep][year][cat][str(mass)]
getattr(cardDict[lep][year][cat][str(mass)],'import')(CBG_Interp)
getattr(cardDict[lep][year][cat][str(mass)],'import')(yieldVar)
cardDict[lep][year][cat][str(mass)].commitTransaction()
testFrame = mzg.frame(float(massList[0])-10,float(massList[-1])+10)
for i,fit in enumerate(fitList):
regionName = fit.GetName().split('_')[-1]
#fit.plotOn(testFrame)
#fit.plotOn(testFrame, RooFit.NormRange('fitRegion_'+regionName))
fit.plotOn(testFrame, RooFit.Normalization(normList[i],RooAbsReal.NumEvent))
for signal in dsList:
signal.plotOn(testFrame)
testFrame.Draw()
testFrame.SetTitle(";m_{H} (GeV);fit pdf")
#c.Print("p5.png")
c.SaveAs(plotBase+"/"+'_'.join(['sig','fit',prod,lep,year,'cat'+cat])+'.png')
for prod in sigNameList:
for mass in massList:
SignalNameParamFixer(year,lep,cat,prod,mass,cardDict[lep][year][cat][mass])
for mass in massList:
fileName = subdir+'/'+'_'.join(['SignalOutput',lep,year,'cat'+cat,mass])
cardDict[lep][year][cat][mass].writeToFile(fileName+'.root')
def ZJPG(f1, c1, globalCut, path):
print "\n\n *** Study of Z/H -> J/Psi Gamma and more ***\n"
print globalCut
u.createDir(path)
c1.cd()
t = f1.Get('apzTree/apzTree')
gStyle.SetMarkerSize(0.6)
gStyle.SetMarkerStyle(20);
gStyle.SetLineWidth(1);
gStyle.SetOptStat(111)
opt = ''
binDownSize = 1
mllCut = TCut('')
name = 'h00-mll-HIG14-003'
m1 = '0'
m2 = '20'
mllCut = TCut('m12<'+m2)
nBins = 50/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m12>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{#mu#mu} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h01-mll-25'
m1 = '0'
m2 = '30'
mllCut = TCut('m12<'+m2)
nBins = 100/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m12>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{#mu#mu} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h02-mll-full'
m1 = '0'
m2 = '150'
mllCut = TCut('m12<'+m2)
nBins = 100/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m12>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{#mu#mu} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h03-mll-jpsi'
m1 = '2.9'
m2 = '3.3'
mllCut = TCut('(m12>'+m1+') && (m12<'+m2+')')
nBins = 20/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m12>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{#mu#mu} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = "h04-mllg-full"
mllg1 = '0'
mllg2 = '200'
mllCut = TCut('')
nBins = 50/binDownSize
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
rangeCut = TCut('(m123>'+mllg1+')&&(m123<'+mllg2+')')
t.Draw('m123>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')', mllCut+globalCut+rangeCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h05-mllg-in_Mll30"
mllg1 = '60'
mllg2 = '150'
m1 = '0'
m2 = '30'
mllCut = TCut('(m12>'+m1+') && (m12<'+m2+')')
nBins = 50/binDownSize
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
rangeCut = TCut('(m123>'+mllg1+')&&(m123<'+mllg2+')')
t.Draw('m123>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')', mllCut+globalCut+rangeCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h06-dr12"
nBins = 50/binDownSize
dr1 = '0'
dr2 = '4'
binWidth = (float(dr2)-float(dr1))/float(nBins)
rangeCut = TCut('(dr12>'+dr1+')&&(dr12<'+dr2+')')
t.Draw('dr12>>'+name+'('+str(nBins)+','+dr1+','+dr2+')', globalCut+rangeCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';#Delta R(#mu_{1}, #mu_{2});Events/%.2f' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h07-dr1234"
nBins = 50/binDownSize
dr1 = '0'
dr2 = '4'
binWidth = (float(dr2)-float(dr1))/float(nBins)
rangeCut = TCut('(dr1234>'+dr1+')&&(dr1234<'+dr2+')')
t.Draw('dr1234>>'+name+'('+str(nBins)+','+dr1+','+dr2+')', globalCut+rangeCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';#Delta R(#mu#mu, #gamma);Events/%.2f' % binWidth)
c1.SaveAs(path+"/"+name+".png")
treeInOne('h08-mllg_full_threeInOne', 'm123', 0, 200, 50, globalCut, t,c1,path)
treeInOne('h09-mllg_mZ_threeInOne', 'm123', 50, 140, 50, globalCut, t,c1,path)
treeInOne('h10-mllg_Hfull_threeInOne','m123',110, 170, 20, globalCut, t,c1,path)
treeInOne('h11-mllg_mH_threeInOne', 'm123',115, 135, 10, globalCut, t,c1,path)
def alphaPiZ2(f1, c1, globalCut, path):
print "\n\n *** Study alpha/piz particle ***\n"
u.createDir(path)
c1.cd()
t = f1.Get('apzTree/apzTree')
gStyle.SetMarkerSize(0.6)
gStyle.SetMarkerStyle(20);
gStyle.SetLineWidth(1);
gStyle.SetOptStat(111)
opt = ''
binDownSize = 1
name = 'h01-m12-full'
mllCut = TCut('')
# myFullCut = TCut(mllCut+globalCut)
m1 = '0'
m2 = '150'
nBins = 100/binDownSize
# print 10*'***', 'Downsized to ', nBins
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m12>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{12} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h02-m34-full'
mllCut = TCut('')
# myFullCut = TCut(mllCut+globalCut)
m1 = '0'
m2 = '150'
nBins = 100/binDownSize
# print 10*'***', 'Downsized to ', nBins
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m34>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{34} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h03-ml12-apz'
mllCut = TCut('')
# myFullCut = TCut(mllCut+globalCut)
m1 = '10'
m2 = '35'
nBins = 15/binDownSize
# print 10*'***', 'Downsized to ', nBins
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m12>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{12} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h04-ml34-apz'
mllCut = TCut('')
# myFullCut = TCut(mllCut+globalCut)
m1 = '10'
m2 = '35'
nBins = 15/binDownSize
# print 10*'***', 'Downsized to ', nBins
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m34>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{12} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h05-m4l_full'
mllCut = TCut('')
m1 = '60'
m2 = '200'
nBins = 30/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m4l>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{4l} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h06-m4l_full'
mllCut = TCut('')
m1 = '100'
m2 = '180'
nBins = 20/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m4l>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{4l} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h07-pt12'
mllCut = TCut('')
m1 = '0'
m2 = '180'
nBins = 50/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('pt12>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';p_{T}^{12} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h08-pt34'
mllCut = TCut('')
m1 = '0'
m2 = '180'
nBins = 50/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('pt34>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';p_{T}^{34} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h09-pt12_over_m4l'
mllCut = TCut('')
m1 = '0'
m2 = '1'
nBins = 50/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('pt12/m4l>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';p_{T}^{12}/m_{4l} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h10-pt34_over_m4l'
mllCut = TCut('')
m1 = '0'
m2 = '1'
nBins = 50/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('pt34/m4l>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';p_{T}^{34}/m_{4l} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h20-m123'
mllCut = TCut('')
m1 = '100'
m2 = '180'
mllCut = TCut('m12<20 && m12>15')
nBins = 20/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m123>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
def shelve(target: str):
target = Path(target)
CraftCore.log.info(f"Creating shelve: {target}")
listFile = configparser.ConfigParser(allow_no_value=True)
updating = target.exists()
if updating:
listFile.read(target, encoding="UTF-8")
oldSections = set(listFile.sections())
if "General" in oldSections:
oldSections.remove("General")
if not listFile.has_section("General"):
listFile.add_section("General")
listFile["General"]["version"] = "2"
blueprintRepos = []
for p in CraftPackageObject.get("craft").children.values():
if p.path in {"craft/craft-core", "craft/craftmaster"}:
continue
blueprintRepos.append(p.instance.repositoryUrl())
listFile["General"]["blueprintRepositories"] = ";".join(blueprintRepos)
reDate = re.compile(r"\d\d\d\d\.\d\d\.\d\d")
def _set(package, key: str, value: str):
if updating:
old = package.get(key, value)
if old != value:
CraftCore.log.info(
f"Updating {package.name} {key}: {old} -> {value}")
package[key] = value
newPackages = set()
for package, version, revision in CraftCore.installdb.getDistinctInstalled(
):
packageObject = CraftPackageObject.get(package)
if not packageObject:
CraftCore.log.warning(
f"{package} is no longer known to Craft, it will not be added to the list"
)
continue
if not packageObject.subinfo.shelveAble:
continue
if not listFile.has_section(package):
listFile.add_section(package)
if updating:
newPackages.add(package)
package = listFile[package]
# TODO: clean our database
patchLvl = version.split("-", 1)
if len(patchLvl) == 2:
# have we encoded a date or a patch lvl?
clean = packageObject.subinfo.options.dailyUpdate and bool(
reDate.match(patchLvl[1]))
clean |= patchLvl[0] in packageObject.subinfo.patchLevel and str(
packageObject.subinfo.patchLevel[patchLvl[0]] +
packageObject.categoryInfo.patchLevel) == patchLvl[1]
if clean:
version = patchLvl[0]
_set(package, "version", version)
if version != packageObject.subinfo.defaultTarget:
CraftCore.debug.printOut(
f"For {packageObject} {version} is an update availible: {packageObject.subinfo.defaultTarget}"
)
if revision:
# sadly we combine the revision with the branch "master-1234ac"
revParts = revision.split("-", 1)
if len(revParts) == 2:
_set(package, "revision", revParts[1])
if updating:
removed = oldSections - newPackages
added = newPackages - oldSections
CraftCore.log.info(
f"The following packages where removed from {target}: {removed}")
CraftCore.log.info(
f"The following packages where added to {target}: {added}")
utils.createDir(target.parent)
listFile._sections = OrderedDict(
sorted(listFile._sections.items(), key=lambda k: k[0]))
with open(target, "wt", encoding="UTF-8") as out:
listFile.write(out)
return True
from level import level1, level2, level3
TXT = '/home/yhb-pc/CNN_Face_keypoint/dataset/test/lfpw_test_249_bbox.txt'
if __name__ == '__main__':
assert(len(sys.argv) == 2)
level = int(sys.argv[1])
if level == 0:
P = partial(level1, FOnly=True)
elif level == 1:
P = level1
elif level == 2:
P = level2
else:
P = level3
OUTPUT = '/home/yhb-pc/CNN_Face_keypoint/dataset/test/out_{0}'.format(level)
createDir(OUTPUT)
data = getDataFromTxt(TXT, with_landmark=False)
for imgPath, bbox in data:
img = cv2.imread(imgPath)
assert(img is not None)
imgGray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
logger("process %s" % imgPath)
landmark = P(imgGray, bbox)
landmark = bbox.reprojectLandmark(landmark)
drawLandmark(img, bbox, landmark)
cv2.imwrite(os.path.join(OUTPUT, os.path.basename(imgPath)), img)
from level import level1, level2, level3
TXT = '/home/yhb-pc/CNN_Face_keypoint/dataset/test/lfpw_test_249_bbox.txt'
if __name__ == '__main__':
assert (len(sys.argv) == 2)
level = int(sys.argv[1])
if level == 0:
P = partial(level1, FOnly=True)
elif level == 1:
P = level1
elif level == 2:
P = level2
else:
P = level3
OUTPUT = '/home/yhb-pc/CNN_Face_keypoint/dataset/test/out_{0}'.format(
level)
createDir(OUTPUT)
data = getDataFromTxt(TXT, with_landmark=False)
for imgPath, bbox in data:
img = cv2.imread(imgPath)
assert (img is not None)
imgGray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
logger("process %s" % imgPath)
landmark = P(imgGray, bbox)
landmark = bbox.reprojectLandmark(landmark)
drawLandmark(img, bbox, landmark)
cv2.imwrite(os.path.join(OUTPUT, os.path.basename(imgPath)), img)
type=str,
default="Lin-784-10 Relu")
parser.add_argument("-loss",
help="loss type",
type=str,
default="Euclidean")
parser.add_argument("-name",
help="experiment name",
type=str,
default="default")
args = parser.parse_args()
# result dir
result_dir = os.path.join("./result", args.name)
plot_dir = os.path.join(result_dir, "plot")
createDir(plot_dir, removeflag=1)
# modify config
for item in args.config.strip().split(" "):
key, value = item.split(":")
assert key in config, "parse error: " + item
config[key] = type(config[key])(value)
# build network
model = Network()
for item in args.arch.strip().split(" "):
parts = item.split("-")
if parts[0] == "Lin":
model.add(Linear('linear', int(parts[1]), int(parts[2]), 0.01))
elif parts[0] == "Relu":
model.add(Relu('relu'))
def enterBuildDir(self):
CraftCore.debug.trace("CraftBase.enterBuildDir called")
utils.createDir(self.buildDir())
os.chdir(self.buildDir())
CraftCore.log.debug("entering: %s" % self.buildDir())
def _replaceSetting(reg,new,text):
return re.compile(reg, re.MULTILINE).sub(new,text)
def resetSettings():
with open( os.path.join( EmergeStandardDirs.etcDir(), "kdesettings.ini.backup"),"rt+") as fin:
text = fin.read()
text = _replaceSetting("^PACKAGE_IGNORES.*$", "PACKAGE_IGNORES =", text)
text = _replaceSetting("^EMERGE_USE_CCACHE.*$", "#EMERGE_USE_CCACHE = True", text)
text = _replaceSetting("^Python =.*$", "Python = C:\python34", text)
text = _replaceSetting("^DOWNLOADDIR =.*$", "#DOWNLOADDIR = C:\kde\download", text)
with open( os.path.join( EmergeStandardDirs.etcDir(), "kdesettings.ini"),"wt+") as fout:
fout.write(text)
if __name__ == "__main__":
EmergeDebug.setVerbose(3)
# we don't use the db directly and the file must not be locked
del InstallDB.installdb
backup()
removeFromDB("dev-util")
removeFromDB("gnuwin32")
# we cant use the ini support to modify the settings as it would kill the comments
resetSettings()
utils.system("cd %s && git clean -xdf" % os.path.join(EmergeStandardDirs.emergeRoot(), "emerge"))
utils.createDir(EmergeStandardDirs.tmpDir())
archiveName = os.path.join(EmergeStandardDirs.tmpDir(), "framewroks-sdk-%s-%s-%s.7z" % (compiler.getCompilerName(), compiler.architecture(), datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d')))
utils.deleteFile(archiveName)
utils.system("7za a %s %s -x!build -x!msys -x!dev-utils -x!tmp -x!mingw* -x!etc/kdesettings.ini.backup -x!etc/portage/install.db.backup" % ( archiveName, EmergeStandardDirs.emergeRoot()))
#utils.copyFile(os.path.join( EmergeStandardDirs.etcDir(), "kdesettings.ini"), os.path.join( EmergeStandardDirs.etcDir(), "kdesettings.ini.backup2"),linkOnly=False)
restore()
period = opt.period
doBkg = opt.bkg
#gROOT.ProcessLine(".L ../tdrstyle.C")
#setTDRStyle()
TH1.SetDefaultSumw2(kTRUE)
if '/tthome' in os.getcwd():
pathBase = "/tthome/andrey/html/zgamma/dalitz/"+ver+"_cut"+cut
hPath = "/tthome/andrey/batch_output/zgamma/8TeV/"+ver
else:
pathBase = "/uscms_data/d2/andreypz/html/zgamma/dalitz/"+ver+"_cut"+cut
hPath = "/eos/uscms/store/user/andreypz/batch_output/zgamma/8TeV/"+ver
u.createDir(pathBase)
if doMerge:
os.system("rm "+hPath+"/m_Data_"+sel+"*.root") #removing the old merged files
dataFile = None
bkgFiles = []
bkgNames = []
u.setSelection(sel)
subsel=sel
if opt.hjp or opt.zjp:
subsel = 'jp-mugamma'
if doMerge:
if sel=="elgamma" or sel=='eegamma':
def doInitialFits(subdir):
print 'loading up the files'
plotBase = '/uscms_data/d2/andreypz/html/zgamma/dalitz/fits-'+subdir+'/init/'
u.createDir(plotBase)
#basePath1 = '/uscms_data/d2/andreypz/zgamma/'+subdir+'/'
basePath1 = '/eos/uscms/store/user/andreypz/batch_output/zgamma/8TeV/'+subdir+'/'
#basePath2 = '/eos/uscms/store/user/andreypz/batch_output/zgamma/8TeV/v30/'
basePath2 = '/uscms_data/d2/andreypz/zgamma/v34-ele2/'
if not os.path.exists(basePath1):
print basePath1, "does not exist!"
sys.exit(0)
if not os.path.exists(basePath1):
print basePath2, "does not exist!"
sys.exit(0)
#musel = "mumu"
musel = "mugamma"
EBetaCut = 1.0
dataDict = {'mu2012':TFile(basePath1+'m_Data_'+musel+'_2012.root','r'),
'el2012':TFile(basePath2+'m_Data_electron_2012.root','r')}
signalDict = {'gg_mu2012_M120':TFile(basePath1+musel+'_2012/hhhh_dal-mad120_1.root','r'),
'gg_mu2012_M125':TFile(basePath1+musel+'_2012/hhhh_dal-mad125_1.root','r'),
'gg_mu2012_M130':TFile(basePath1+musel+'_2012/hhhh_dal-mad130_1.root','r'),
'gg_mu2012_M135':TFile(basePath1+musel+'_2012/hhhh_dal-mad135_1.root','r'),
'gg_mu2012_M140':TFile(basePath1+musel+'_2012/hhhh_dal-mad140_1.root','r'),
'gg_mu2012_M145':TFile(basePath1+musel+'_2012/hhhh_dal-mad145_1.root','r'),
'gg_mu2012_M150':TFile(basePath1+musel+'_2012/hhhh_dal-mad150_1.root','r'),
'vbf_mu2012_M120':TFile(basePath1+musel+'_2012/hhhh_vbf-mad120_1.root','r'),
'vbf_mu2012_M125':TFile(basePath1+musel+'_2012/hhhh_vbf-mad125_1.root','r'),
'vbf_mu2012_M130':TFile(basePath1+musel+'_2012/hhhh_vbf-mad130_1.root','r'),
'vbf_mu2012_M135':TFile(basePath1+musel+'_2012/hhhh_vbf-mad135_1.root','r'),
'vbf_mu2012_M140':TFile(basePath1+musel+'_2012/hhhh_vbf-mad140_1.root','r'),
'vbf_mu2012_M145':TFile(basePath1+musel+'_2012/hhhh_vbf-mad145_1.root','r'),
'vbf_mu2012_M150':TFile(basePath1+musel+'_2012/hhhh_vbf-mad150_1.root','r'),
'gg_el2012_M125':TFile(basePath2+'electron_2012/hhhh_dal-mad125_1.root','r')}
treeName = 'fitTree/fitTree'
weight = RooRealVar('Weight','Weight',0,100)
lowCutOff = 100
highCutOff = 200
mzg = RooRealVar('CMS_hzg_mass','CMS_hzg_mass', lowCutOff,highCutOff)
mzg.setRange('FullRegion', lowCutOff, highCutOff)
mzg.setRange('DalitzRegion', 100,200)
mzg.setBins(50000,'cache')
c = TCanvas("c","c",0,0,500,400)
c.cd()
ws =RooWorkspace("ws")
# ###################################
# start loop over all year/lep/cat #
# ###################################
for year in yearList:
for lepton in leptonList:
for cat in catList:
if rootrace:
print "rootrace"
RooTrace.dump()
raw_input()
signalList = []
signalListDH = []
signalListPDF = []
if verbose: print 'top of loop',year,lepton,cat
# ##################################################
# set up the signal histograms and the mzg ranges #
# ##################################################
for prod in sigNameList:
signalListDS = []
for mass in massList:
# store the unbinned signals for CB fitting
signalTree = signalDict[prod+"_"+lepton+year+"_M"+mass].Get(treeName)
#signalTree.Print()
sigName = '_'.join(['ds_sig',prod,lepton,year,'cat'+cat,'M'+mass])
sig_argSW = RooArgSet(mzg,weight)
sig_ds = RooDataSet(sigName,sigName,sig_argSW,'Weight')
lumiWeight = LumiXSWeighter(int(mass), prod,lepton)
print signalTree, "A signal tree", prod, ' categ=',cat, "mass =", mass
for i in signalTree:
if cat=="EB" and fabs(i.ph_eta)>EBetaCut: continue
elif cat=="EE" and fabs(i.ph_eta)<EBetaCut: continue
if i.m_llg> lowCutOff and i.m_llg<highCutOff:
mzg.setVal(i.m_llg)
sigWeight = lumiWeight*i.weight
#sigWeight = lumiWeight
sig_ds.add(sig_argSW, sigWeight)
#sig_argSW.Print()
#raw_input()
signalListDS.append(sig_ds)
getattr(ws,'import')(signalListDS[-1])
#signalTree.ResetBranchAddresses()
# do some histogramming for gg signal for bias study
# we don't need or use unbinned signal or complicated fits
# but this is mostly for compatibility, we may change to unbinned
# during a future iteration
if prod=='gg':
if verbose: print 'signal mass loop', mass
print 'in gg INPUT ', cat, prod,mass
#raw_input()
histName = '_'.join(['sig',lepton,year,'cat'+cat,'M'+mass])
rangeName = '_'.join(['range',lepton,year,'cat'+cat,'M'+mass])
signalList.append(TH1F(histName, histName, 100, lowCutOff, highCutOff))
signalList[-1].SetLineColor(kRed)
signalTree = signalDict[prod+"_"+lepton+year+"_M"+mass].Get(treeName)
if verbose:
print histName
signalTree.Print()
print
if cat=="0":
signalTree.Draw('m_llg>>'+histName,'weight')
elif cat=="EB":
signalTree.Draw('m_llg>>'+histName,"weight*(fabs(ph_eta)<"+str(EBetaCut)+")")
elif cat=="EE":
signalTree.Draw('m_llg>>'+histName,"weight*(fabs(ph_eta)>"+str(EBetaCut)+")")
if signalList[-1].Integral()!=0:
signalList[-1].Scale(1/signalList[-1].Integral())
signalList[-1].Smooth(2)
# range is +/- 1 RMS centered around signal peak
rangeLow = signalList[-1].GetMean()-1.0*signalList[-1].GetRMS()
rangeHi = signalList[-1].GetMean()+1.0*signalList[-1].GetRMS()
mzg.setRange(rangeName,rangeLow,rangeHi)
mzg_argL = RooArgList(mzg)
mzg_argS = RooArgSet(mzg)
signalListDH.append(RooDataHist('dh_'+histName, 'dh_' +histName,mzg_argL,signalList[-1]))
signalListPDF.append(RooHistPdf('pdf_'+histName,'pdf_'+histName,mzg_argS,signalListDH[-1],2))
getattr(ws,'import')(signalListPDF[-1])
if verbose: print '\n\n ** finshed one mass -->>', mass
if debugPlots and prod=='gg':
testFrame = mzg.frame()
for i,signal in enumerate(signalListPDF):
signalListDH[i].plotOn(testFrame)
signal.plotOn(testFrame)
testFrame.Draw()
c.SaveAs(plotBase+'_'.join(['signals',prod,year,lepton,'cat'+cat,'M'+mass])+'.png')
if debugPlots:
testFrame = mzg.frame()
for signal in signalListDS:
signal.plotOn(testFrame, RooFit.DrawOption('pl'))
testFrame.Draw()
c.SaveAs(plotBase+'_'.join(['ds','sig',prod,year,lepton,'cat'+cat,'M'+mass])+'.png')
del signalTree
# ###############
# get the data #
# ###############
if verbose: print 'starting data section'
dataName = '_'.join(['data',lepton,year,'cat'+cat])
dataTree = dataDict[lepton+year].Get(treeName)
data_argS = RooArgSet(mzg)
data_ds = RooDataSet(dataName,dataName,data_argS)
for i in dataTree:
if cat=="EB" and fabs(i.ph_eta)>EBetaCut: continue
elif cat=="EE" and fabs(i.ph_eta)<EBetaCut: continue
if i.m_llg> lowCutOff and i.m_llg<highCutOff:
mzg.setVal(i.m_llg)
data_ds.add(data_argS)
#dataTree.ResetBranchAddresses()
if verbose:
print dataName
data_ds.Print()
print
if debugPlots:
testFrame = mzg.frame()
data_ds.plotOn(testFrame,RooFit.Binning(50))
testFrame.Draw()
c.SaveAs(plotBase+'_'.join(['data',year,lepton,'cat'+cat,'M'+mass])+'.png')
getattr(ws,'import')(data_ds)
# ############
# make fits #
# ############
if verbose: 'starting fits'
if cat!='5':
#GaussExp = BuildGaussExp(year, lepton, cat, mzg)
#SechExp = BuildSechExp(year, lepton, cat, mzg)
#GaussBern3 = BuildGaussStepBern3(year, lepton, cat, mzg)
#GaussBern4 = BuildGaussStepBern4(year, lepton, cat, mzg)
#GaussBern5 = BuildGaussStepBern5(year, lepton, cat, mzg)
#GaussBern6 = BuildGaussStepBern6(year, lepton, cat, mzg)
Exp = BuildExp(year, lepton, cat, mzg)
Pow = BuildPow(year, lepton, cat, mzg)
Bern2 = BuildBern3(year, lepton, cat, mzg)
Bern3 = BuildBern3(year, lepton, cat, mzg)
Bern4 = BuildBern4(year, lepton, cat, mzg)
Bern5 = BuildBern5(year, lepton, cat, mzg)
Bern6 = BuildBern6(year, lepton, cat, mzg)
#SechPow = BuildSechPow(year, lepton, cat, mzg)
#GaussPow = BuildGaussPow(year, lepton, cat, mzg)
#GaussBern4 = BuildGaussStepBern4(year, lepton, cat, mzg, step = 105, stepLow = 100, stepHigh = 150, sigma = 2.5)
#GaussBern5 = BuildGaussStepBern5(year, lepton, cat, mzg, step = 105, stepLow = 100, stepHigh = 150, sigma = 2.5)
#GaussBern6 = BuildGaussStepBern6(year, lepton, cat, mzg, step = 105, stepLow = 90, stepHigh = 190, sigma = 2.5)
#SechBern3 = BuildSechStepBern3(year, lepton, cat, mzg)
#if lepton == 'mu' and cat == '3': SechBern4 = BuildSechStepBern4(year, lepton, cat, mzg,sigma=2)
##else: SechBern4 = BuildSechStepBern4(year, lepton, cat, mzg)
#if lepton == 'mu' and cat == '3': SechBern5 = BuildSechStepBern5(year, lepton, cat, mzg,sigma=2)
#else: SechBern5 = BuildSechStepBern5(year, lepton, cat, mzg)
#gauss = BuildRooGaussian(year, lepton, cat, mzg)
#BetaFunc = BuildBetaFunc(year, lepton, cat, mzg, 'DalitzRegion')
#Kumaraswamy = BuildKumaraswamy(year, lepton, cat, mzg, 'DalitzRegion')
#BB = BuildBetaAndBern(year, lepton, cat, mzg, 'DalitzRegion')
#GB = BuildGaussAndBern(year, lepton, cat, mzg, 'DalitzRegion')
if verbose:
Exp.Print()
#GaussExp.Print()
#GaussPow.Print()
#SechExp.Print()
#SechPow.Print()
#GaussBern3.Print()
#GaussBern4.Print()
#GaussBern5.Print()
#GaussBern6.Print()
#GaussExp.fitTo(data_ds,RooFit.Range('FullRegion'))
#SechExp.fitTo(data_ds,RooFit.Range('FullRegion'))
#GaussBern3.fitTo(data_ds,RooFit.Range('FullRegion'))
#GaussBern4.fitTo(data_ds,RooFit.Range('FullRegion'))
#GaussBern5.fitTo(data_ds,RooFit.Range('FullRegion'))
#GaussBern6.fitTo(data_ds,RooFit.Range('FullRegion'))
Exp.fitTo(data_ds,RooFit.Range('DalitzRegion'))
Pow.fitTo(data_ds,RooFit.Range('DalitzRegion'))
Bern2.fitTo(data_ds,RooFit.Range('DalitzRegion'))
Bern3.fitTo(data_ds,RooFit.Range('DalitzRegion'))
Bern4.fitTo(data_ds,RooFit.Range('DalitzRegion'))
Bern5.fitTo(data_ds,RooFit.Range('DalitzRegion'))
Bern6.fitTo(data_ds,RooFit.Range('DalitzRegion'))
#GaussPow.fitTo(data_ds,RooFit.Range('FullRegion'))
#SechPow.fitTo(data_ds,RooFit.Range('FullRegion'))
#SechBern3.fitTo(data_ds,RooFit.Range('FullRegion'))
#SechBern4.fitTo(data_ds,RooFit.Range('FullRegion'))
#SechBern5.fitTo(data_ds,RooFit.Range('FullRegion'))
#gauss.fitTo(data_ds,RooFit.Range('DalitzRegion'))
#BetaFunc.fitTo(data_ds,RooFit.Range('FullRegion'))
#Kumaraswamy.fitTo(data_ds,RooFit.Range('FullRegion'))
#BB.fitTo(data_ds,RooFit.Range('FullRegion'))
#GB.fitTo(data_ds,RooFit.Range('FullRegion'))
if debugPlots:
leg = TLegend(0.7,0.7,1.0,1.0)
leg.SetFillColor(0)
leg.SetShadowColor(0)
leg.SetBorderSize(1)
leg.SetHeader('_'.join(['test','fits',year,lepton,'cat'+cat]))
testFrame = mzg.frame(RooFit.Range('DalitzRegion'))
data_ds.plotOn(testFrame, RooFit.Binning(50))
#GaussExp.plotOn(testFrame,RooFit.Name('GaussExp'))
#SechExp.plotOn(testFrame,RooFit.LineColor(kRed), RooFit.Name('SechExp'))
#GaussBern3.plotOn(testFrame,RooFit.LineColor(kGreen), RooFit.Name('GaussBern3'))
#GaussBern4.plotOn(testFrame,RooFit.LineColor(kPink), RooFit.Name('GaussBern4'))
#GaussBern5.plotOn(testFrame,RooFit.LineColor(kOrange), RooFit.Name('GaussBern5'))
#GaussBern6.plotOn(testFrame,RooFit.LineColor(kPink), RooFit.Name('GaussBern6'))
Exp.plotOn(testFrame,RooFit.LineColor(colors['exp']), RooFit.Name('Exp'))
Exp.plotOn(testFrame,RooFit.LineColor(colors['pow']), RooFit.Name('Pow'))
Bern2.plotOn(testFrame,RooFit.LineColor(colors['bern2']), RooFit.Name('Bern2'))
Bern3.plotOn(testFrame,RooFit.LineColor(colors['bern3']), RooFit.Name('Bern3'))
Bern4.plotOn(testFrame,RooFit.LineColor(colors['bern4']), RooFit.Name('Bern4'))
Bern5.plotOn(testFrame,RooFit.LineColor(colors['bern5']), RooFit.Name('Bern5'))
Bern6.plotOn(testFrame,RooFit.LineColor(colors['bern6']), RooFit.Name('Bern6'))
#GaussPow.plotOn(testFrame,RooFit.LineColor(kCyan), RooFit.Name('GaussPow'))
#SechPow.plotOn(testFrame,RooFit.LineColor(kOrange), RooFit.Name('SechPow'))
#SechBern3.plotOn(testFrame,RooFit.LineColor(kMagenta))
#SechBern4.plotOn(testFrame,RooFit.LineColor(kBlack))
#SechBern5.plotOn(testFrame,RooFit.LineColor(kGreen))
#gauss.plotOn(testFrame,RooFit.LineColor(kBlue), RooFit.Name('Gauss'))
#BetaFunc.plotOn(testFrame,RooFit.LineColor(kBlack), RooFit.Name('Beta'))
#Kumaraswamy.plotOn(testFrame,RooFit.LineColor(kCyan), RooFit.Name('Kumaraswamy'))
#BB.plotOn(testFrame,RooFit.LineColor(kViolet), RooFit.Name('Beta+Bern4'))
#GB.plotOn(testFrame,RooFit.LineColor(kGreen), RooFit.Name('GaussBern3'))
testFrame.Draw()
testFrame.SetTitle(";m_{H} (GeV);Events/2 GeV")
leg.AddEntry(testFrame.findObject('Exp'),'Exp','l')
leg.AddEntry(testFrame.findObject('Pow'),'Pow','l')
leg.AddEntry(testFrame.findObject('Bern2'),'Bern2','l')
leg.AddEntry(testFrame.findObject('Bern3'),'Bern3','l')
leg.AddEntry(testFrame.findObject('Bern4'),'Bern4','l')
leg.AddEntry(testFrame.findObject('Bern5'),'Bern5','l')
leg.AddEntry(testFrame.findObject('Bern6'),'Bern6','l')
#leg.AddEntry(testFrame.findObject('Beta'),'Beta','l')
#leg.AddEntry(testFrame.findObject('Kumaraswamy'),'Kumaraswamy','l')
#leg.AddEntry(testFrame.findObject('Beta+Bern4'),'Beta+Bern4','l')
#leg.AddEntry(testFrame.findObject('GaussPow'),'GaussPow','l')
#leg.AddEntry(testFrame.findObject('SechPow'), 'SechPow', 'l')
#leg.AddEntry(testFrame.findObject('GaussExp'),'GaussExp','l')
#leg.AddEntry(testFrame.findObject('SechExp'), 'SechExp', 'l')
#leg.AddEntry(testFrame.findObject('GaussBern3'),'GaussBern3','l')
#leg.AddEntry(testFrame.findObject('GaussBern4'),'GaussBern4','l')
#leg.AddEntry(testFrame.findObject('GaussBern5'),'GaussBern5','l')
#leg.AddEntry(testFrame.findObject('GaussBern6'),'GaussBern6','l')
leg.Draw()
c.Print(plotBase+'_'.join(['fits',year,lepton,'cat'+cat])+'.png')
#raw_input()
#getattr(ws,'import')(GaussExp)
#getattr(ws,'import')(SechExp)
getattr(ws,'import')(Exp)
getattr(ws,'import')(Pow)
getattr(ws,'import')(Bern2)
getattr(ws,'import')(Bern3)
getattr(ws,'import')(Bern4)
getattr(ws,'import')(Bern5)
getattr(ws,'import')(Bern6)
#getattr(ws,'import')(GaussBern3)
#getattr(ws,'import')(GaussBern4)
#getattr(ws,'import')(GaussBern5)
#getattr(ws,'import')(GaussBern6)
#getattr(ws,'import')(SechBern3)
#getattr(ws,'import')(SechBern4)
#getattr(ws,'import')(SechBern5)
#getattr(ws,'import')(GB)
#getattr(ws,'import')(GaussPow)
#getattr(ws,'import')(SechPow)
else:
print cat
ws.commitTransaction()
u.createDir(subdir)
ws.writeToFile(subdir+'/testRooFitOut_Dalitz.root')
ws.Print()
print '\n \t we did it!\t'
def generate_hdf5_data(filelist, output, fname, argument=False):
data = getDataFromTxt(filelist)
F_imgs = []
F_landmarks = []
EN_imgs = []
EN_landmarks = []
NM_imgs = []
NM_landmarks = []
for (imgPath, bbox, landmarkGt) in data:
img = cv2.imread(imgPath, cv2.CV_LOAD_IMAGE_GRAYSCALE)
assert (img is not None)
logger("process %s" % imgPath)
# Paper Table2 jitter F-layer
f_bbox = bbox.subBBox(-0.05, 1.05, -0.05, 1.05)
f_face = img[f_bbox.top:f_bbox.bottom + 1,
f_bbox.left:f_bbox.right + 1]
## data argument
if argument and np.random.rand() > -1:
### flip
face_flipped, landmark_flipped = flip(f_face, landmarkGt)
face_flipped = cv2.resize(face_flipped, (39, 39))
F_imgs.append(face_flipped.reshape((1, 39, 39)))
F_landmarks.append(landmark_flipped.reshape(10))
### rotation +5 degrees
if np.random.rand() > 0.5:
face_rotated_by_alpha, landmark_rotated = rotate(img, f_bbox, \
bbox.reprojectLandmark(landmarkGt), 5)
landmark_rotated = bbox.projectLandmark(landmark_rotated)
face_rotated_by_alpha = cv2.resize(face_rotated_by_alpha,
(39, 39))
F_imgs.append(face_rotated_by_alpha.reshape((1, 39, 39)))
F_landmarks.append(landmark_rotated.reshape(10))
### flip with rotation
face_flipped, landmark_flipped = flip(face_rotated_by_alpha,
landmark_rotated)
face_flipped = cv2.resize(face_flipped, (39, 39))
F_imgs.append(face_flipped.reshape((1, 39, 39)))
F_landmarks.append(landmark_flipped.reshape(10))
### rotation -5 degrees
if np.random.rand() > 0.5:
face_rotated_by_alpha, landmark_rotated = rotate(img, f_bbox, \
bbox.reprojectLandmark(landmarkGt), -5)
landmark_rotated = bbox.projectLandmark(landmark_rotated)
face_rotated_by_alpha = cv2.resize(face_rotated_by_alpha,
(39, 39))
F_imgs.append(face_rotated_by_alpha.reshape((1, 39, 39)))
F_landmarks.append(landmark_rotated.reshape(10))
### flip with rotation
face_flipped, landmark_flipped = flip(face_rotated_by_alpha,
landmark_rotated)
face_flipped = cv2.resize(face_flipped, (39, 39))
F_imgs.append(face_flipped.reshape((1, 39, 39)))
F_landmarks.append(landmark_flipped.reshape(10))
f_face = cv2.resize(f_face, (39, 39))
en_face = f_face[:31, :]
nm_face = f_face[8:, :]
f_face = f_face.reshape((1, 39, 39))
f_landmark = landmarkGt.reshape((10))
F_imgs.append(f_face)
F_landmarks.append(f_landmark)
## data argument for EN
if argument and np.random.rand() > 0.5:
### flip
face_flipped, landmark_flipped = flip(en_face, landmarkGt)
face_flipped = cv2.resize(face_flipped, (31, 39)).reshape(
(1, 31, 39))
landmark_flipped = landmark_flipped[:3, :].reshape((6))
EN_imgs.append(face_flipped)
EN_landmarks.append(landmark_flipped)
en_face = cv2.resize(en_face, (31, 39)).reshape((1, 31, 39))
en_landmark = landmarkGt[:3, :].reshape((6))
EN_imgs.append(en_face)
EN_landmarks.append(en_landmark)
## data argument for NM
if argument and np.random.rand() > 0.5:
### flip
face_flipped, landmark_flipped = flip(nm_face, landmarkGt)
face_flipped = cv2.resize(face_flipped, (31, 39)).reshape(
(1, 31, 39))
landmark_flipped = landmark_flipped[2:, :].reshape((6))
NM_imgs.append(face_flipped)
NM_landmarks.append(landmark_flipped)
nm_face = cv2.resize(nm_face, (31, 39)).reshape((1, 31, 39))
nm_landmark = landmarkGt[2:, :].reshape((6))
NM_imgs.append(nm_face)
NM_landmarks.append(nm_landmark)
#Convert the list to array
F_imgs, F_landmarks = np.asarray(F_imgs), np.asarray(F_landmarks)
EN_imgs, EN_landmarks = np.asarray(EN_imgs), np.asarray(EN_landmarks)
NM_imgs, NM_landmarks = np.asarray(NM_imgs), np.asarray(NM_landmarks)
### normalize the data and shu
F_imgs = processImage(F_imgs)
shuffle_in_unison_scary(F_imgs, F_landmarks)
EN_imgs = processImage(EN_imgs)
shuffle_in_unison_scary(EN_imgs, EN_landmarks)
NM_imgs = processImage(NM_imgs)
shuffle_in_unison_scary(NM_imgs, NM_landmarks)
# full face
base = join(OUTPUT, '1_F')
createDir(base)
output = join(base, fname)
logger("generate %s" % output)
with h5py.File(output, 'w') as h5:
h5['data'] = F_imgs.astype(np.float32)
h5['landmark'] = F_landmarks.astype(np.float32)
# eye and nose
base = join(OUTPUT, '1_EN')
createDir(base)
output = join(base, fname)
logger("generate %s" % output)
with h5py.File(output, 'w') as h5:
h5['data'] = EN_imgs.astype(np.float32)
h5['landmark'] = EN_landmarks.astype(np.float32)
# nose and mouth
base = join(OUTPUT, '1_NM')
createDir(base)
output = join(base, fname)
logger("generate %s" % output)
with h5py.File(output, 'w') as h5:
h5['data'] = NM_imgs.astype(np.float32)
h5['landmark'] = NM_landmarks.astype(np.float32)
def downloadHistoricalData(ticker_list):
utils.createDir(config.STOCKS_DATA_FOLDER)
if _outdatedData():
_retrieveStockHistorical(ticker_list)
_retrieveStockDetailedHistorical(ticker_list)
#sigZip = zip(['H #rightarrow #mu#mu#gamma'],
# [twoFile])
#u.drawAllInFile(oneFile, '', None, sigZip, 'LHE', '', path, None,"norm", isLog=True)
#myzip = zip(['5<mll<20'],[testFile])
#u.drawAllInFile(oneFile, "0<mll<2", myzip, twoFile, '2<mll<5', '', path, None,"norm", isLog=True)
#u.drawAllInFile(oneFile, "ggH to eeg", '', twoFile, 'VH to eeg', '', path, None,"norm", isLog=True)
#u.drawAllInFile(oneFile, "h #rightarrow ee#gamma", '', twoFile, 'h #rightarrow #mu#mu#gamma', '', path, None,"norm", isLog=False)
#u.drawAllInFile(oneFile, "3 GeV", [('15 GeV',twoFile)],testFile, '35 GeV', '', path, None,"norm", isLog=True)
#u.drawAllInFile(oneFile, "DYG", [('DYJ',twoFile)],testFile, 'Sig', '', path, None,"norm", isLog=True)
u.createDir(pathBase+"/csBR")
c1 = TCanvas("c1","c1", 600,500)
h1 = oneFile.Get("LHE_diLep_mass_bins")
h2 = twoFile.Get("LHE_diLep_mass_bins")
#h.Print('all')
h1.Draw()
h2.Draw('same')
h2.SetLineColor(kBlue)
c1.SaveAs(pathBase+"/csBR/dilepmass.png")
hel = oneFile.Get("LHE_diLep_mass_low2")
hmu = twoFile.Get("LHE_diLep_mass_low2")
iEL1 = hel.Integral(0,105)
import socket
hostname = socket.gethostname()
if '/tthome' in os.getcwd():
pathBase = "/tthome/andrey/html/zgamma/dalitz/"+ver+"_cut"+cut
hPath = "/tthome/andrey/batch_output/zgamma/8TeV/"+ver
elif 'cmslpc' in hostname:
pathBase = "/uscms_data/d2/andreypz/html/zgamma/dalitz/"+ver+"_cut"+cut
hPath = "/eos/uscms/store/user/andreypz/batch_output/zgamma/8TeV/"+ver
elif 'pcncu' in hostname or 'lxplus' in hostname:
pathBase = "/afs/cern.ch/user/a/andrey/work/html/"+ver+"_cut"+cut
hPath = os.getcwd()+"/"+ver
u.createDir(pathBase)
dataFile = None
bkgFiles = []
bkgNames = []
u.setSelection(sel)
subsel=sel
qcdSamples = None
for bg in opt.bkg:
print '\t Adding bkg=', bg
#bkgFiles.append(TFile(hPath+"/m_ZG_"+subsel+"_"+period+".root","OPEN"))
#bkgNames.append('ZG')
if bg in ['zjets','all']:
## number(0.16/0.18) froms table 2 in paper
types = [
(0, 'LE1', 0.16),
(0, 'LE2', 0.18),
(1, 'RE1', 0.16),
(1, 'RE2', 0.18),
(2, 'N1', 0.16),
(2, 'N2', 0.18),
(3, 'LM1', 0.16),
(3, 'LM2', 0.18),
(4, 'RM1', 0.16),
(4, 'RM2', 0.18),
]
for t in types:
d = 'train/2_%s' % t[1]
createDir(d)
def generate(ftxt, mode, argument=False):
"""
Generate Training Data for LEVEL-2
mode = train or test
"""
data = getDataFromTxt(ftxt)
trainData = defaultdict(lambda: dict(patches=[], landmarks=[]))
for (imgPath, bbox, landmarkGt) in data:
img = cv2.imread(imgPath, cv2.CV_LOAD_IMAGE_GRAYSCALE)
assert (img is not None)
logger("process %s" % imgPath)
if args.verbose is not None:
if args.verbose == "debug":
verbosity = logging.DEBUG
elif args.verbose == "warning":
verbosity = logging.WARNING
elif args.verbose == "error":
verbosity = logging.ERROR
outdir = os.getcwd() + "/" + VAR_ID
if args.output:
outdir = args.output
if outdir.endswith("/"):
outdir = outdir[:-1]
if not utils.createDir(outdir):
LOGGER.error("Could not create output dir %s" % outdir)
sys.exit(1)
config.OUTPUT_DIR = outdir
if GWAVA is not None:
if GWAVA.endswith("/"):
GWAVA = GWAVA[:-1]
config.GWAVA_DIR = GWAVA
# ------------------------------------------------------------------------------------------------------------------------
LOGGER = logging.getLogger("annotator")
LOGGER.setLevel(verbosity)
ch = logging.StreamHandler()
l.SetLineColor(kRed+2)
l.SetLineWidth(2)
#l.SetLineStyle(21)
l.Draw()
elif opt.mll:
leg.SetY1NDC(0.64)
leg.AddEntry(standardM,"10 #times SM", "lp")
for e in ['.png', '.pdf']:
CMS_lumi(c, 2, 11, "")
limitDir = plotBase+'/Limits'
if opt.br:
limitDir = plotBase+'/Limits-xsBr'
u.createDir(limitDir)
if opt.mll and opt.divide:
c.SaveAs(limitDir+'/limits_Mll_dM'+e)
elif opt.mll:
c.SaveAs(limitDir+'/limits_Mll'+e)
else:
if opt.br:
c.SaveAs(limitDir+'/limits_xsBR_cat_'+suff+e)
else:
c.SaveAs(limitDir+'/limits_cat_'+suff+e)
out.cd()
observed.Write("observed")
expected.Write("expected")
oneSigma.Write("oneSigma")
def getFiles(url,
extCompile,
recursionDepth=5,
httpDomain=utils.HTTPS_DOMAIN,
baseDir=None):
# Args: url, extCompile=> A pattern object of the extension(s) to match
# recursionDepth => An integer that indicates how deep to scrap
# Note: A negative recursion depth indicates that you want
# to keep crawling as far as the program can go
if not recursionDepth:
return
elif not hasattr(extCompile, 'search'):
utils.streamPrintFlush(
"Expecting a pattern object/result of re.compile(..) for arg 'extCompile'\n",
sys.stderr)
return
if not utils.httpHeadCompile.search(url):
url = "%s%s" % (httpDomain, url)
decodedData = utils.dlAndDecode(url)
if not decodedData:
return
else:
urls = utils.urlCompile.findall(decodedData)
urls = list(
map(
lambda s: utils.repeatHttpHeadCompile.sub(
utils.HTTP_HEAD_REGEX, s), urls))
if not urls:
capableUrls = utils.urlCapableCompile.findall(decodedData)
trimmedHeadUrl = url.strip('/')
for capableUrl in capableUrls:
trimmed = capableUrl.strip('/')
fixedUrl = '%s/%s' % (trimmedHeadUrl, trimmed)
urls.append(fixedUrl)
plainUrls = []
matchedFileUrls = []
for u in urls:
pathSelector = plainUrls
regSearch = extCompile.search(u)
if regSearch:
g = regSearch.groups(1)
u = '%s.%s' % (g[0], g[1])
pathSelector = matchedFileUrls
pathSelector.append(u)
if not baseDir:
baseDir = os.path.abspath(".")
fullUrlToMemPath = os.path.join(baseDir,
utils.pathCleanseCompile.sub('_', url))
utils.createDir(fullUrlToMemPath)
# Time to download all the matched files
dlResults = []
for eachUrl in matchedFileUrls:
dlResults.append(dlData(eachUrl, fullUrlToMemPath))
resultsList = list(filter(lambda val: val, dlResults))
# Report to user successful saves
downloadCount = len(resultsList)
# print(downloadCount)
if not downloadCount:
# Mark this url as a bad one/miss and for the sake of crawling
# not hitting dead ends, we won't crawl it anymore unless otherwise specified
urlHash = getHash(url)
urlScoreTuple = missesDict.get(urlHash, None)
badCrawlCount = 0
if urlScoreTuple and len(urlScoreTuple) != 2:
badCrawlCount = (
urlScoreTuple[1]) + 1 # Increment the bad crawl score
missesDict[urlHash] = (url, badCrawlCount, time.time())
return # Cut this journey short
else:
utils.streamPrintFlush(
"For url %s downloaded %d files\n" % (url, downloadCount),
sys.stderr)
recursionDepth -= 1
for eachUrl in plainUrls:
getFiles(eachUrl,
extCompile,
recursionDepth,
baseDir=fullUrlToMemPath)
cf.read('config.cfg')
subdir = cf.get("path","ver")
yearList = [a.strip() for a in (cf.get("fits","yearList")).split(',')]
leptonList = [a.strip() for a in (cf.get("fits","leptonList")).split(',')]
catList = [a.strip() for a in (cf.get("fits","catList")).split(',')]
sigNameList = [a.strip() for a in (cf.get("fits","sigNameList")).split(',')]
doBlind = int(cf.get("fits","blind"))
hjp = 0
if 'hjp' in sigNameList:
hjp = 1
leptonList = ['mu']
mllBins = u.mllBins()
plotBase = cf.get("path","htmlbase")+"/html/zgamma/dalitz/fits-"+subdir
u.createDir(plotBase+'/BestFits')
rooWsFile = TFile(subdir+'/testRooFitOut_Dalitz.root')
myWs = rooWsFile.Get('ws')
card_ws = RooWorkspace('ws_card')
card_ws.autoImportClassCode(True)
c = TCanvas("c","c",0,0,500,400)
c.cd()
mzg = myWs.var('CMS_hzg_mass')
mzg.setRange('signal',120,130)
mzg.setRange('r1',110,120)
mzg.setRange('r2',130,170)
if hjp:
bkgModel = 'Bern2'
from utils import getDataFromTxt, createDir, logger, shuffle_in_unison_scary, processImage, getPatch, randomShiftWithArgument
## number(0.16/0.18) froms table 2 in paper
types = [(0, 'LE1', 0.16),
(0, 'LE2', 0.18),
(1, 'RE1', 0.16),
(1, 'RE2', 0.18),
(2, 'N1', 0.16),
(2, 'N2', 0.18),
(3, 'LM1', 0.16),
(3, 'LM2', 0.18),
(4, 'RM1', 0.16),
(4, 'RM2', 0.18),]
for t in types:
d = 'train/2_%s' % t[1]
createDir(d)
def generate(ftxt, mode, argument=False):
"""
Generate Training Data for LEVEL-2
mode = train or test
"""
data = getDataFromTxt(ftxt)
trainData = defaultdict(lambda: dict(patches=[], landmarks=[]))
for (imgPath, bbox, landmarkGt) in data:
img = cv2.imread(imgPath, cv2.CV_LOAD_IMAGE_GRAYSCALE)
assert(img is not None)
logger("process %s" % imgPath)
landmarkPs = randomShiftWithArgument(landmarkGt, 0.05)
def alphaPiZ(f1, c1, globalCut, path):
print "study alpha/piz particle"
u.createDir(path)
c1.cd()
t = f1.Get('fitTree/fitTree')
gStyle.SetMarkerSize(0.6)
gStyle.SetMarkerStyle(20);
gStyle.SetLineWidth(1);
gStyle.SetOptStat(111)
opt = ''
binDownSize = 1
if 'alphaPiZ-3' in path:
binDownSize = 2
elif 'alphaPiZ-4' in path:
binDownSize = 2
elif 'alphaPiZ-6' in path:
binDownSize = 2
name = 'h00-mll-HIG14-003'
mllCut = TCut('')
# myFullCut = TCut(mllCut+globalCut)
m1 = '0'
m2 = '20'
nBins = 50/binDownSize
# print 10*'***', 'Downsized to ', nBins
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m_ll>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{#mu#mu} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h01-mll-full'
mllCut = TCut('')
# myFullCut = TCut(mllCut+globalCut)
m1 = '0'
m2 = '25'
nBins = 100/binDownSize
# print 10*'***', 'Downsized to ', nBins
binWidth = (float(m2)-float(m1))/float(nBins)
t.Draw('m_ll>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.SetTitle(name+';m_{#mu#mu} (GeV);Events/%.2f GeV' % binWidth)
h.UseCurrentStyle()
c1.SaveAs(path+"/"+name+".png")
name = 'h02-mll-jpsi'
m1 = '2.7'
m2 = '4.0'
nBins = 100/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
t.Draw('m_ll>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h03-mll-upsilon"
m1 = '9.0'
m2 = '12.0'
nBins = 30/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
t.Draw('m_ll>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h04-mll-apz"
m1 = '15.0'
m2 = '25.0'
nBins = 30/binDownSize
binWidth = (float(m2)-float(m1))/float(nBins)
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
t.Draw('m_ll>>'+name+'('+str(nBins)+','+m1+','+m2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h05-mllg-full"
mllg1 = '60'
mllg2 = '150'
m1 = '0'
m2 = '25'
nBins = 50/binDownSize
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
t.Draw('m_llg>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h06-mllg-jpsi-full"
mllg1 = '100'
mllg2 = '150'
m1 = '3.0'
m2 = '3.2'
nBins = 20/binDownSize
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
mllCut = TCut('m_ll>'+m1+' && m_ll<'+m2)
t.Draw('m_llg>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')',mllCut*globalCut, opt)
# print 3*"***", 'my integrall =', h.Integral()
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h07-mllg-jpsi-Z"
mllg1 = '70'
mllg2 = '110'
m1 = '3.0'
m2 = '3.2'
nBins = 20/binDownSize
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
t.Draw('m_llg>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h08-mllg-upsilon-Z"
mllg1 = '70'
mllg2 = '110'
m1 = '9.2'
m2 = '9.6'
nBins = '30'
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
t.Draw('m_llg>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h09-mllg-apz-full"
mllg1 = '70'
mllg2 = '200'
m1 = '17.0'
m2 = '19.0'
nBins = 100/binDownSize
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
t.Draw('m_llg>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h10-mllg-apz-full"
mllg1 = '100'
mllg2 = '150'
m1 = '17.0'
m2 = '19.0'
nBins = 20/binDownSize
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
mllCut = TCut('m_ll>'+m1+' && m_ll<'+m2)
# print 15*"*", 'myCut is = ', myFullCut
t.Draw('m_llg>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')', mllCut+globalCut, opt)
# t.Draw('m_llg>>'+name, mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h11-mllg-apz-Z"
mllg1 = '70'
mllg2 = '110'
m1 = '17'
m2 = '19'
nBins = 40/binDownSize
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
myFullCut = TCut(mllCut+globalCut)
t.Draw('m_llg>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h12-phEta-apz"
m1 = '17.0'
m2 = '19.0'
nBins = 50/binDownSize
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
myFullCut = TCut(mllCut+globalCut)
t.Draw('fabs(ph_eta)>>'+name+'('+str(nBins)+',0,3)', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';|#eta_{#gamma}|;Events')
c1.SaveAs(path+"/"+name+".png")
name = "h13-mllg-apz-125"
mllg1 = '115'
mllg2 = '135'
m1 = '17.0'
m2 = '19.0'
nBins = 30/binDownSize
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
myFullCut = TCut(mllCut+globalCut)
t.Draw('m_llg>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h14-mllg-jpsi-125"
mllg1 = '115'
mllg2 = '135'
myFullCut = TCut(mllCut+globalCut)
m1 = '3.0'
m2 = '3.2'
nBins = 30/binDownSize
binWidth = (float(mllg2)-float(mllg1))/float(nBins)
mllCut = TCut('(m_ll>'+m1+') && (m_ll<'+m2+')')
t.Draw('m_llg>>'+name+'('+str(nBins)+','+mllg1+','+mllg2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';m_{#mu#mu#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h15-ptgamma-apz"
pt1='40'
pt2='120'
m1 = '17.0'
m2 = '19.0'
nBins = 30/binDownSize
binWidth = (float(pt2)-float(pt1))/float(nBins)
mllCut = TCut('m_ll>'+m1+' && m_ll<'+m2)
myFullCut = TCut(mllCut+globalCut)
t.Draw('ph_pt>>'+name+'('+str(nBins)+','+pt1+','+pt2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';p_{T}^{#gamma} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h16-ptll-apz"
pt1='40'
pt2='120'
m1 = '17.0'
m2 = '19.0'
nBins = 30/binDownSize
binWidth = (float(pt2)-float(pt1))/float(nBins)
mllCut = TCut('m_ll>'+m1+' && m_ll<'+m2)
myFullCut = TCut(mllCut+globalCut)
t.Draw('di_pt>>'+name+'('+str(nBins)+','+pt1+','+pt2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';p_{T}^{#mu#mu} (GeV);Events/%.2f GeV' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h17-ptgamma-overMllg-apz"
pt1='0.3'
pt2='1.0'
m1 = '17.0'
m2 = '19.0'
nBins = 30/binDownSize
binWidth = (float(pt2)-float(pt1))/float(nBins)
mllCut = TCut('m_ll>'+m1+' && m_ll<'+m2)
myFullCut = TCut(mllCut+globalCut)
t.Draw('ph_pt/m_llg>>'+name+'('+str(nBins)+','+pt1+','+pt2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';p_{T}^{#gamma}/m_{#mu#mu#gamma};Events/%.2f bin' % binWidth)
c1.SaveAs(path+"/"+name+".png")
name = "h18-ptll-overMllg-apz"
pt1='0.3'
pt2='1.0'
m1 = '17.0'
m2 = '19.0'
nBins = 30/binDownSize
binWidth = (float(pt2)-float(pt1))/float(nBins)
mllCut = TCut('m_ll>'+m1+' && m_ll<'+m2)
myFullCut = TCut(mllCut+globalCut)
t.Draw('di_pt/m_llg>>'+name+'('+str(nBins)+','+pt1+','+pt2+')', mllCut+globalCut, opt)
h = gDirectory.Get(name)
h.Draw("same e1p")
h.UseCurrentStyle()
h.SetTitle(name+';p_{T}^{#mu#mu}/m_{#mu#mu#gamma};Events/%.2f bin' % binWidth)
c1.SaveAs(path+"/"+name+".png")
experiment_dir = utils.createExperimentDir(experiment)
if not experiment_dir:
print("Already have an experiment with this name...")
exit()
# load dataset
dataset = np.load(dataset_name)
# dataset = dataset[:1000]
dataset_size = len(dataset)
vae, encoder, decoder, beta = model.build()
if load:
filename = load
vae.load_weights(os.path.join(experiment_dir, filename + ".h5"))
experiment_dir = os.path.join(experiment_dir, str(time.time()))
utils.createDir(experiment_dir)
# save model info
utils.saveModelDescription(encoder, decoder, experiment_dir)
# set training
def warmup(epoch):
threshold_epoch = 5
if epoch > threshold_epoch:
value = 0.001
ref_epoch = epoch - threshold_epoch
value = value + ref_epoch * 0.00001
value = min(1, value)
else:
value = 0
if fp:
fp.close()
return isSuccess
if __name__ == "__main__":
# Parse arguments
parser = argparse.ArgumentParser(
description="Upload TMDB TV show data to S3",
add_help=True
)
parser.add_argument("path_config", type=str,
help="Path to configuration file with API credentials")
args = parser.parse_args()
# Create the required directories if not exits
if not createDir(dirLogs):
sys.exit('The directory "{}" could not be created'.format(dirLogs))
if not createDir(dirTweet):
sys.exit('The directory "{}" could not be created'.format(dirTweet))
# Setup the logger
logName = date.today().strftime("%Y-%m-%d") + '-tweet-upload.log'
setupLogger(dirLogs, logName)
# Get datetime interval for the past hour
dt_save, datetime_start, datetime_end = getPastHourInterval()
# Connect to the database
conn = connectToDB(dbPath)
if conn is None:
logging.error('Error while connecting to the database')
def makeCards(subdir):
yearList = [a.strip() for a in (cf.get("fits","yearList")).split(',')]
leptonList = [a.strip() for a in (cf.get("fits","leptonList")).split(',')]
catList = [a.strip() for a in (cf.get("fits","catList")).split(',')]
if hjp: leptonList =['mu']
for year in yearList:
for lep in leptonList:
for cat in catList:
if lep=='el' and cat!='EB': continue
bgFileName = subdir+'/testCardBackground_Dalitz.root'
bgFile = TFile(bgFileName)
bgWs = bgFile.Get('ws_card')
bgWs.Print()
channel = '_'.join([lep,year,'cat'+cat])
print channel
#bkgParams = ['sigma','mean','tau','norm']
if hjp:
bkgParams = ['p1','p2','norm']
else:
bkgParams = ['p1','p2','p3','p4','norm']
for mass in massList:
if cat in ['m1','m2','m3','m4','m5','m6','m7'] and (mass!='125.0' or not opt.br): continue
sigFileName = subdir+'/'+'_'.join(['SignalOutput',lep,year,'cat'+cat,mass])+'.root'
sigFile = TFile(sigFileName)
sigWs = sigFile.Get('ws_card')
#sigWs.Print()
procList = []
for s in sigNameList:
if lep=='el' and s=='v': continue
elif cat in ['m1','m2','m3','m4','m5','m6','m7'] and s!='gg': continue
elif s=='v':
procList.extend(['WH','ZH'])
else:
procList.append(proc[s])
if opt.br:
if hjp:
procList=['hjp']
else:
procList=['ggH']
print 'List of procs:', procList
nProc = len(procList)
# print procList
#if opt.br and float(mass)%5!=0:
# print "Sorry can't do this with those mass points yet, ",mass
# sys.exit(0)
# croDict = {a:float(u.conf.get(a+"H-"+mass[0:3], "cs-mu")) for a in sigNameList}
# print croDict
if opt.br: outCardDir = '/output_cards_xsbr/'
else: outCardDir = '/output_cards/'
u.createDir(subdir+outCardDir)
card = open(subdir+outCardDir+'_'.join(['hzg',lep,year,'cat'+cat,'M'+mass,'Dalitz',brTag])+'.txt','w')
card.write(introMessage)
card.write('imax *\n')
card.write('jmax *\n')
card.write('kmax *\n')
card.write('---------------\n')
card.write('shapes {0:<8} * {1:<20} ws_card:$PROCESS_$CHANNEL\n'.format('*',bgFileName))
card.write('shapes {0:<8} * {1:<20} ws_card:bkg_$CHANNEL\n'.format('bkg',bgFileName))
for sig in sigNameList:
if sig=='v' and lep=='el': continue
if opt.br and sig!='gg' and sig!='hjp': continue
elif sig=='v':
card.write('shapes {0:<8} * {1:<20} ws_card:{2}_$CHANNEL\n'.format('ZH',sigFileName,'sig_'+sig))
card.write('shapes {0:<8} * {1:<20} ws_card:{2}_$CHANNEL\n'.format('WH',sigFileName,'sig_'+sig))
else:
card.write('shapes {0:<8} * {1:<20} ws_card:{2}_$CHANNEL\n'.format(proc[sig],sigFileName,'sig_'+sig))
card.write('---------------\n')
bgYield = bgWs.var('data_yield_'+channel).getVal()
print cat, "bg yield:", bgYield
bkgRate = 1
#if Ext: bkgRate = '1'
#else: bkgRate = str(bgYield)
card.write('{0:<12} {1}\n'.format('bin',channel))
card.write('{0:<12} {1}\n'.format('observation',int(bgYield)))
card.write('------------------------------\n')
#print channel, prefixSigList, prefixSigList[::-1]
nSubLine1 = '{0:<25}'
nSubLine2 = '{0:<30}'
for i in xrange(nProc):
nSubLine1 +=' {'+str(i+1)+':^15}'
nSubLine2 +=' {'+str(i+1)+':^5}'
# print i, nSubLine1
nSubLine1 +=' {'+str(i+2)+':^15} \n'
nSubLine2 +=' {'+str(i+2)+':5} - \n'
# print 'nSubLine1=', nSubLine1
card.write(nSubLine1.format(*(['bin']+[channel]*(nProc+1))))
card.write(nSubLine1.format(*(['process']+procList[::-1]+['bkg'])))
card.write(nSubLine1.format(*(['process']+range(-(nProc-1), 2, 1))))
card.write('--------------------------------------------------------------\n')
sigYields = []
for s in sigNameList[::-1]:
if lep=='el' and s=='v': continue
if opt.br and s!='gg' and s!='hjp': continue
cs = 1
if opt.br:
if hjp:
cs = u.getCS("HtoJPsiGamma")
else:
if lep=='mu': cs = fitMu(float(mass))
elif lep=='el': cs = fitEl(float(mass))
else: print 'This fit does not exist'
if cat in ['m1','m2','m3','m4','m5','m6','m7']:
cs = cs*mllBins[int(cat[1])][1]
# When doing limit on cs*BR, need to divide by cs:
procYield = sigWs.var('sig_'+s+'_yield_'+channel).getVal()/cs
print '\t Category:', cat, ' signal:', s, 'm=', mass
print '\t ** From the fit: %.3f' % cs
print mass, s, lep, 'cs for scale=', cs
else:
procYield = sigWs.var('sig_'+s+'_yield_'+channel).getVal()
print mass, s, lep
if opt.br:
print '\t \t raw sig= ', procYield*cs, 'Acc*Eff = %.3f' % (procYield/lumi2012)
else:
# cs = u.getCS('%s-%i' %(proc_conf[s], float(mass)), lep)
print '\t \t raw sig= ', procYield, 'Acc*Eff = %.3f' % (procYield/lumi2012/cs)
if s=='v':
zh_frac = float(xsDict[YR][TeV]['ZH'][mass])/(float(xsDict[YR][TeV]['ZH'][mass])+float(xsDict[YR][TeV]['WH'][mass]))
wh_frac = 1-zh_frac
print s, 'YR=',YR, ' ZH fraction =', zh_frac, ' WH fraction =',wh_frac
sigYields.append('%.4f'%(procYield*zh_frac))
sigYields.append('%.4f'%(procYield*wh_frac))
else:
sigYields.append('%.4f'%procYield)
print 'M =', mass, 'YIELDS=', sigYields
sigRate = sigYields
card.write(nSubLine1.format(*(['rate']+sigRate+[bkgRate])))
card.write('-------------------------------------------------------------\n')
card.write(' \n')
card.write(nSubLine2.format(*(['lumi_8TeV', 'lnN']+nProc*[lumi])))
mmm = mass
if float(mass)>140:
# after mH=140 the syst only available with 1GeV intervals
mmm = mass[0:4]+'0'
if not opt.br:
card.write(nSubLine2.format(*(['CMS_hllg_brLLG', 'lnN']+nProc*[brLLG])))
if lep=='el':
card.write('pdf_qqbar lnN '+xsPDFErrDict['YR3'][yearToTeV[year]]['VBF'][mmm]+' - - \n')
card.write('QCDscale_qqH lnN '+xsScaleErrDict['YR3'][yearToTeV[year]]['VBF'][mmm]+' - - \n')
card.write('pdf_gg lnN - '+xsPDFErrDict['YR3'][yearToTeV[year]]['ggF'][mmm]+' - \n')
card.write('QCDscale_ggH lnN - '+xsScaleErrDict['YR3'][yearToTeV[year]]['ggF'][mmm]+' - \n')
else:
card.write('QCDscale_ZH lnN '+xsScaleErrDict['YR3'][yearToTeV[year]]['ZH'][mmm]+' - - - - \n')
card.write('QCDscale_WH lnN - '+xsScaleErrDict['YR3'][yearToTeV[year]]['WH'][mmm]+' - - - \n')
card.write('pdf_qqbar lnN '+
xsPDFErrDict['YR3'][yearToTeV[year]]['ZH'][mmm]+' '+
xsPDFErrDict['YR3'][yearToTeV[year]]['WH'][mmm]+' '+
xsPDFErrDict['YR3'][yearToTeV[year]]['VBF'][mmm]+' - - \n')
card.write('QCDscale_qqH lnN - - '+xsScaleErrDict['YR3'][yearToTeV[year]]['VBF'][mmm]+' - - \n')
card.write('pdf_gg lnN - - - '+xsPDFErrDict['YR3'][yearToTeV[year]]['ggF'][mmm]+' - \n')
card.write('QCDscale_ggH lnN - - - '+xsScaleErrDict['YR3'][yearToTeV[year]]['ggF'][mmm]+' - \n')
if lep=='mu':
card.write(nSubLine2.format(*(['CMS_eff_m', 'lnN']+nProc*[muID])))
#card.write(nSubLine2.format(*(['CMS_eff_m_ISO', 'lnN']+nProc*[muISO])))
card.write(nSubLine2.format(*(['CMS_eff_m_MuEgTRIG', 'lnN']+nProc*[muTRIG])))
card.write(nSubLine2.format(*(['CMS_eff_g', 'lnN']+nProc*[phoID])))
card.write(nSubLine2.format(*(['CMS_eff_g_MuEgTRIG', 'lnN']+nProc*[phoTRIG_mu])))
else:
card.write(nSubLine2.format(*(['CMS_eff_e', 'lnN']+nProc*[elID])))
card.write(nSubLine2.format(*(['CMS_eff_g', 'lnN']+nProc*[phoID])))
card.write(nSubLine2.format(*(['CMS_eff_g_GGTRIG', 'lnN']+nProc*[phoTRIG_el])))
card.write(nSubLine2.format(*(['CMS_hllg_PU', 'lnN']+nProc*[PU])))
for sig in sigNameList:
if lep=='el' and sig=='v': continue
card.write('{0:<40} {1:<10} {2:^10} {3:^10}\n'.format('sig_'+sig+'_mShift_' +channel,'param', 1, meanUnc[lep]))
card.write('{0:<40} {1:<10} {2:^10} {3:^10}\n'.format('sig_'+sig+'_sigmaShift_'+channel,'param', 1, sigmaUnc[lep]))
for param in bkgParams[:-1]:
card.write('{0:<45} {1:<15}\n'.format('bkg_'+param+'_'+channel,'flatParam'))
card.write('{0:<45} {1:<15}\n'.format('bkg_'+channel+'_'+bkgParams[-1],'flatParam'))
card.close()
def SignalFitMaker(lep, year, cat, subdir):
f = TFile('../data/Dalitz_BR50.root','READ')
g = f.Get('csbr_mu')
xsBr = g.GetFunction('pol4')
#g.Print('all')
hfile = TFile('hists.root','recreate')
hists = HistManager(hfile)
#print massList
u.set_palette()
# reasd these from a config file:
sigNameList = [a.strip() for a in (cf.get("fits","sigNameList")).split(',')]
tev = u.yearToTeV(year)
if 'hjp' in sigNameList:
if lep!='mu': return
massList0 = ['125']
massList = ['125.0']
#print massList
#raw_input()
plotBase = cf.get("path","htmlbase")+"/html/zgamma/dalitz/fits-"+subdir+'/init' + '_'.join([year,lep,'cat'+cat])+'/'
u.createDir(plotBase)
rooWsFile = TFile(subdir+'/testRooFitOut_Dalitz.root')
myWs = rooWsFile.Get('ws')
#myWs.Print()
RooRandom.randomGenerator().SetSeed(8675309)
c = TCanvas("c","c",0,0,500,400)
c.cd()
mzg = myWs.var('CMS_hzg_mass')
paraNames = {}
cardDict = u.AutoVivification()
mzg.setRange('statRange125',120,130)
global massList0, massList
#print massList, massList0
masses0 = massList0
masses1 = massList
if cat in ['m1','m2','m3','m4','m5','m6','m7']:
masses0 = ['125']
masses1 = ['125.0']
for mass in masses1:
cardDict[lep][year][cat][mass] = RooWorkspace('ws_card')
print cardDict[lep][year][cat][mass]
# we need crystal ball + gaussian fits for all mass points, and for all production methods.
# we also need to interpolate the distributions for 0.5 mass bins, so we use some tricks
# in order to create new fits out of the existing 5GeV steps
# ##################
# Start the Loop! #
# ##################
for prod in sigNameList:
if lep=='el' and prod=='v': continue
if cat in ['m1','m2','m3','m4','m5','m6','m7']:
if prod!='gg': continue
# First of all make the fits to the existing MC samples
SigFits = {'0':None} # Format 'mass': fit-function
dsList = []
for m in masses0:
fitBuilder = FitBuilder(mzg, year,lep,cat,sig=prod, mass=m+'.0')
mzg.setRange('fitRegion1',int(m)-11,int(m)+11)
sigName = '_'.join(['ds','sig',prod,lep,year,'cat'+cat,'M'+m])
sig_ds = myWs.data(sigName)
dsList.append(sig_ds)
SigFits[m],paramList = fitBuilder.Build(myFunc, piece = 'Interp', mean = float(m))
SigFits[m].fitTo(sig_ds, RooFit.Range('fitRegion1'), RooFit.SumW2Error(kTRUE),
RooFit.Strategy(1), RooFit.NumCPU(4), RooFit.PrintLevel(-1))
# print paramList
# raw_input("\n ---> Param List. hit enter to continue")
paraNames[prod] = []
for param in paramList:
param.setConstant(True)
print param.GetName(), param.getVal()
if 'sigmaCBCBG' in param.GetName():
n = '_'.join(['sigmaCBCBG',year,lep,'cat'+cat,prod])
hists.fill1DHist(param.getVal(), n,";sigmaCB", 50, 0,5, 1, '')
elif 'fracCBG' in param.GetName():
n = '_'.join(['fracCBS',year,lep,'cat'+cat,prod])
hists.fill1DHist(param.getVal(), n,";fracCBG", 50, 0,1, 1, '')
elif 'mean' in param.GetName():
n = '_'.join(['mean',year,lep,'cat'+cat,prod])
hists.fill1DHist(param.getVal(), n,";mean", 160, 115,155, 1, '')
elif 'sigmaGCBG' in param.GetName():
n = '_'.join(['sigmaGCBS',year,lep,'cat'+cat,prod])
hists.fill1DHist(param.getVal(), n,";sigmaGCBG", 50, 0,5, 1, '')
else:
n = None
paraNames[prod].append(n)
#r.Print()
#a = r.floatParsFinal()
#a[0].Print()
#print a[0].GetName(), a[0].getVal()
#raw_input('RooFitResults above.')
fitList = []
normList = []
oldMassHi = oldMassLow = 0
for massString in masses1:
fitBuilder = FitBuilder(mzg, year,lep,cat,sig=prod,mass=massString)
#fitBuilder.__init__(mzg,year,lep,cat,sig=prod,mass=str(massLow))
# ############################################
# Get the high and low mass references #
# If the point does not need interpolation, #
# we just use it for high and low #
# ############################################
mass = float(massString)
if mass%5.0 == 0.0:
massHi = int(mass)
massLow = int(mass)
else:
massRound = roundTo5(massString)
if mass<massRound:
massHi = massRound
massLow = massRound-5
else:
massHi = massRound+5
massLow = massRound
if not(oldMassLow == massLow and oldMassHi == massHi):
# only calc the low and high points if they change
oldMassLow = massLow
oldMassHi = massHi
# print 'massLow and massHi=', massLow, massHi
# raw_input("Mass low and Hi"+massString+"; hit enter")
massDiff = (massHi - mass)/5.
sigName = '_'.join(['ds','sig',prod,lep,year,'cat'+cat,'M'+str(massLow)])
sig_ds_Low = myWs.data(sigName)
sigName = '_'.join(['ds','sig',prod,lep,year,'cat'+cat,'M'+str(massHi)])
sig_ds_Hi = myWs.data(sigName)
if massHi==massLow:
# No interpolateion needed, just take the existing fit
SigFit_Interp = SigFits[str(mass)[:3]]
else:
# ##### interpolate the two mass points
# print 'massdiff=',massDiff
# raw_input('Massdiff...')
mzg.setRange('fitRegion_'+massString,mass-11,mass+11)
#mzg.setRange('fitRegion_'+massString,mass-5,mass+5)
beta = RooRealVar('beta','beta', 0.5, 0., 1.)
beta.setVal(massDiff)
SigFit_Low = SigFits[str(massLow)]
SigFit_Hi = SigFits[str(massHi)]
interp_pdf = RooIntegralMorph('interp_pdf', 'interp_pdf', SigFit_Low, SigFit_Hi, mzg, beta)
interp_ds = interp_pdf.generate(RooArgSet(mzg), 10000)
fitBuilder.__init__(mzg,year,lep,cat,sig=prod,mass=str(mass))
SigFit_Interp,paramList = fitBuilder.Build(myFunc, piece = 'Interp', mean = mass)
SigFit_Interp.fitTo(interp_ds, RooFit.Range('fitRegion_'+massString), RooFit.SumW2Error(kTRUE),
RooFit.Strategy(1), RooFit.NumCPU(4), RooFit.PrintLevel(-1))
# r.Print()
# raw_input('RooFitResults above.')
for param in paramList:
param.setConstant(True)
if 'sigmaCBCBG' in param.GetName():
n = '_'.join(['sigmaCBCBG',year,lep,'cat'+cat,prod])
hists.fill1DHist(param.getVal(), n,";sigmaCB", 50, 0.5,2.5, 1, '')
elif 'fracCBG' in param.GetName():
n = '_'.join(['fracCBS',year,lep,'cat'+cat,prod])
hists.fill1DHist(param.getVal(), n,";fracCBG", 50, 0,0.5, 1, '')
elif 'sigmaGCBG' in param.GetName():
n = '_'.join(['sigmaGCBS',year,lep,'cat'+cat,prod])
hists.fill1DHist(param.getVal(), n,";sigmaGCBG", 50, 0,5, 1, '')
else:
n = None
normList.append(sig_ds_Low.sumEntries()*massDiff+sig_ds_Hi.sumEntries()*(1-massDiff))
#acceff_low = (sig_ds_Low.sumEntries()/(xsBr(massLow)*lumi))
#acceff_Hi = (sig_ds_Hi.sumEntries()/(xsBr(massHi)*lumi))
#yie_calc = xsBr(mass)*lumi*(massDiff*acceff_low + (1-massDiff)*acceff_Hi)
#print lep,year,cat,mass, cardDict[lep][year][cat][str(mass)]
#print "\n\n Signal Yield?? = ", normList[-1], yie_calc
#print "Low and Hi, sumentries:", sig_ds_Low.sumEntries(), sig_ds_Hi.sumEntries()
#print 'Low and Hi acc*eff:', acceff_low, acceff_Hi
#if massLow!=massHi:
# raw_input('Yields for signals ')
yieldName = '_'.join(['sig',prod,'yield',lep,year,'cat'+cat])
#yieldVar = RooRealVar(yieldName,yieldName, yie_calc)
yieldVar = RooRealVar(yieldName,yieldName, normList[-1])
# sigNameInterp = '_'.join(['ds','sig',prod,lep,year,'cat'+cat,'M'+str(mass)])
fitList.append(SigFit_Interp)
getattr(cardDict[lep][year][cat][str(mass)],'import')(SigFit_Interp)
getattr(cardDict[lep][year][cat][str(mass)],'import')(yieldVar)
cardDict[lep][year][cat][str(mass)].commitTransaction()
gStyle.SetOptStat(1)
for n in paraNames[prod]:
if n==None: continue
s = hfile.Get(n)
#s.Print()
#raw_input('s print')
s.Draw('hist')
c.SaveAs(plotBase+'params_'+n+'.png')
gStyle.SetOptStat(1)
testFrame = mzg.frame(float(masses1[0])-10,float(masses1[-1])+10)
for signal in dsList:
signal.plotOn(testFrame, RooFit.MarkerStyle(6))
# print signal.mean(mzg)
# raw_input("Mean of the fit ")
for i,fit in enumerate(fitList):
regionName = fit.GetName().split('_')[-1]
# fit.plotOn(testFrame)
# fit.plotOn(testFrame, RooFit.NormRange('fitRegion_'+regionName))
fit.plotOn(testFrame, RooFit.NormRange('fitRegion_'+regionName), RooFit.Normalization(normList[i],RooAbsReal.NumEvent),
RooFit.LineColor(TColor.GetColorPalette(i*10)))
# fit.paramOn(testFrame)
# testFrame.getAttText().SetTextSize(0.02)
# testFrame.getAttText().SetTextColor(kRed)
sigmaDS = []
for i,signal in enumerate(dsList):
#if i==0: m=125
#if i==1: m=145
# reduced = signal.reduce("CMS_hzg_mass>"+str(m*(1-0.1))+"&&CMS_hzg_mass<"+str(m*(1+0.1)))
# reduced.plotOn(testFrame, RooFit.MarkerStyle(20+i), RooFit.MarkerSize(1), RooFit.Binning(150))
# sigmaDS.append(reduced.sigma(mzg))
signal.plotOn(testFrame, RooFit.MarkerStyle(20+i), RooFit.MarkerSize(1), RooFit.Binning(150))
# signal.statOn(testFrame,RooFit.What("MR"),RooFit.Layout(0.60,0.83,0.87),RooFit.CutRange('statRange125'))
print signal.mean(mzg), signal.sigma(mzg)
# raw_input("Mean and sigma from dataset above")
m = testFrame.GetMaximum()
testFrame.SetMaximum(1.1*m)
testFrame.SetMinimum(0.005)
'''
if prod == 'gg':
testFrame.SetMaximum(1.1*m)
elif prod =='vbf':
testFrame.SetMaximum(0.05)
elif prod =='v':
testFrame.SetMaximum(0.03)
'''
testFrame.SetMinimum(1e-4)
testFrame.Draw()
if lep=="el":
testFrame.SetTitle(";m_{ee#gamma} (GeV);Signal shape")
else:
testFrame.SetTitle(";m_{#mu#mu#gamma} (GeV);Signal shape")
'''
lat = TLatex()
lat.SetNDC()
lat.SetTextSize(0.045)
#lat.DrawLatex(0.66,0.85, 'Mean = %.1f'%dsList[0].mean(mzg))
lat.DrawLatex(0.42,0.80, '\\frac{RMS}{m_{H}} = \\frac{%.2f}{125} = %.3f' %( sigmaDS[0], sigmaDS[0]/125) )
lat.DrawLatex(0.62,0.60, '\\frac{RMS}{m_{H}} = \\frac{%.2f}{145} = %.3f' %( sigmaDS[1], sigmaDS[1]/145) )
'''
CMS_lumi(c, 2, 11, "Simulation")
c.SaveAs(plotBase+'_'.join(['sig','fit',prod,lep,year,'cat'+cat])+'.png')
for prod in sigNameList:
if lep=='el' and prod=='v': continue
if cat in ['m1','m2','m3','m4','m5','m6','m7']:
if prod!='gg': continue
for mass in masses1:
SignalNameParamFixer(year,lep,cat,prod,mass,cardDict[lep][year][cat][mass])
#print cardDict[lep][year][cat][mass]
#raw_input("RAW INPUT")
#file = TFile("parampampam.root",'recreate')
#file.cd()
#mass="121.0"
#mean = RooRealVar("mean","mean", 130,120,140)
#sigma = RooRealVar("sigma","sigma", 5, 0.1,30.0)
#fff = RooGaussian("fff","gauss model",mzg, mean, sigma)
#fff = (cardDict[lep][year][cat][mass]).pdf('_'.join(['sig',prod,lep,year,'cat'+cat]))
#fff.fitTo(sig_ds_Low, RooFit.Range(110, 150))
#fff.fitTo(sig_ds_Low, RooFit.Range(110, 130), RooFit.SumW2Error(kTRUE), RooFit.Strategy(1), RooFit.NumCPU(4), RooFit.PrintLevel(-1))
#testFrame = mzg.frame(110,150)
#dsList[0].plotOn(testFrame)
#fff.plotOn(testFrame)
#fff.paramOn(testFrame, RooFit.Layout(0.18,0.43,0.67))
#testFrame.Draw()
##c.SaveAs(plotBase+"/"+'_'.join(['refit_sig','fit',prod,lep,year,'cat'+cat])+'.png')
#c.Write()
for mass in masses1:
fileName = subdir+'/'+'_'.join(['SignalOutput',lep,year,'cat'+cat,mass])
cardDict[lep][year][cat][mass].writeToFile(fileName+'.root')
if __name__ == "__main__":
# Parse arguments
parser = argparse.ArgumentParser(
description="Upload TMDB TV show data to S3", add_help=True)
parser.add_argument("path_config",
type=str,
help="Path to configuration file with API credentials")
parser.add_argument("path_tv_show",
type=str,
help="Path to the CSV file with TV show information")
args = parser.parse_args()
# Create the required directories if not exits
if not createDir(dirLogs):
sys.exit('The directory "{}" could not be created'.format(dirLogs))
if not createDir(dirDB):
sys.exit('The directory "{}" could not be created'.format(dirDB))
# Setup the logger
logName = date.today().strftime("%Y-%m-%d") + '-tweet-stream.log'
setupLogger(dirLogs, logName)
# Connect to the database
conn = connectToDB(dbPath)
if conn is None:
logging.error('Error while connecting to the database')
sys.exit(1)
cursor = conn.cursor()
