def doMake():
hasLib = os.path.isdir(glob.libDirName)
hasExe = os.path.isfile(glob.exeName)
isClean = glob.pars["clean"]
isMake = glob.pars["make"] or (not hasLib) or (not hasExe)
if isClean or isMake:
resetDir(glob.libDirName, isClean)
if isMake:
log.info(
blue(" - Moving to ") + red(glob.libDirName) +
blue(" and compiling ANNZ... "))
mkfl = os.path.join(glob.annzDir, 'Makefile')
cmnd = "cd " + glob.libDirName + " ; make " + glob.makeOpt + " -f " + mkfl
cmkdStatus = os.system(cmnd)
Assert("compilation failed", (cmkdStatus == 0))
if os.path.isfile(glob.exeName):
log.info(
blue(" - Found ") + red(glob.exeName) +
blue(" - compilation seems to have succeded... "))
# check that the executable exists before moving on and add the lib dir to LD_LIBRARY_PATH (needed on some systems)
if not isClean:
Assert("Did not find ANNZ executable (" + glob.exeName + ")",
os.path.isfile(glob.exeName))
if not glob.libDirName in os.environ["LD_LIBRARY_PATH"]:
if os.environ["LD_LIBRARY_PATH"] == "":
os.environ["LD_LIBRARY_PATH"] = glob.libDirName
else:
os.environ["LD_LIBRARY_PATH"] = glob.libDirName + ":" + os.environ[
"LD_LIBRARY_PATH"]
if glob.pars["onlyMake"]: exit(0)
def initROOT():
if "ROOTSYS" in os.environ and glob.useDefinedROOTSYS:
log.info(
blue(" - Found defined ") + green("ROOTSYS") + blue(" = \"") +
yellow(os.environ["ROOTSYS"]) +
blue("\". Setting glob.rootHome to match"))
glob.rootHome = os.environ["ROOTSYS"]
rootExeDir = glob.rootHome + "/bin/"
rootExe = rootExeDir + "/root"
rootHomeLib = glob.rootHome + "/lib/"
Assert("Found rootHome = "+glob.rootHome+" which does not exist... Please set this in commonImports.py to the" \
+" ROOT installation directory !",os.path.isdir(glob.rootHome))
Assert("Did not find ROOT executable, expected as "+rootExe+" ..." \
+" Is ROOT properly installed ? Is glob.rootHome set to match ?",os.path.isfile(rootExe))
# add the bin directory of ROOT as first in the PATH, to make sure that the correct bin/root-config is used in the Makefile
os.environ["PATH"] = rootExeDir + ":" + os.environ["PATH"]
if not "ROOTSYS" in os.environ:
os.environ["ROOTSYS"] = glob.rootHome
log.info(
blue(" - Setting ") + green("ROOTSYS") + blue(" = \"") +
yellow(os.environ["ROOTSYS"]) + blue("\""))
elif glob.rootHome != os.environ["ROOTSYS"]:
os.environ["ROOTSYS"] = glob.rootHome
log.info(
blue(" - Setting ") + green("ROOTSYS") + blue(" = \"") +
yellow(os.environ["ROOTSYS"]) + blue("\""))
else:
log.info(
blue(" - Will use ") + green("ROOTSYS") + blue(" = \"") +
yellow(os.environ["ROOTSYS"]) + blue("\""))
if not "LD_LIBRARY_PATH" in os.environ:
os.environ["LD_LIBRARY_PATH"] = rootHomeLib
log.info(
blue(" - Setting ") + green("LD_LIBRARY_PATH") + blue(" = \"") +
yellow(os.environ["LD_LIBRARY_PATH"]) + blue("\""))
elif not (rootHomeLib) in os.environ["LD_LIBRARY_PATH"]:
os.environ["LD_LIBRARY_PATH"] = rootHomeLib + ":" + os.environ[
"LD_LIBRARY_PATH"]
log.info(
blue(" - Adding to ") + green("LD_LIBRARY_PATH") + blue(" \"") +
yellow(rootHomeLib) + blue("\""))
else:
log.info(
blue(" - Found ") + green("LD_LIBRARY_PATH") + blue(" = \"") +
yellow(os.environ["LD_LIBRARY_PATH"]) + blue("\""))
return
def runANNZ():
exitStatus = runOneANNZ()
if exitStatus == 0: return
# if ANNZ failed during training, try again with a different random seed and randomized MLM options
# --------------------------------------------------------------------------------------------------
if glob.annz["doTrain"]:
# set random seed if not already set
setSeed = (not glob.annz.has_key("initSeedRnd"))
if not setSeed: setSeed = (glob.annz["initSeedRnd"] < 1)
if setSeed: glob.annz["initSeedRnd"] = 198876
# remove all userMLMopts so that a random set is generated
glob.annz["userMLMopts"] = ""
if glob.annz["doBinnedCls"]:
for nRndOptNow in range(glob.annz["binCls_nTries"]):
glob.annz.pop("userMLMopts_" + str(nRndOptNow), None)
nTries = 0
for nTryNow in range(nTries):
glob.annz["initSeedRnd"] += 1
log.warning(
whtOnRed(" - runANNZ failed !!! Will try again (" +
str(nTryNow + 1) + "/" + str(nTries) +
") with initSeedRnd = ") +
yellowOnRed(str(glob.annz["initSeedRnd"])))
exitStatus = runOneANNZ()
if exitStatus == 0: break
Assert("runANNZ failed !!!", (exitStatus == 0))
return
def resetDir(dirName, resetOutDir, verb=True):
Assert("Tried to resetDir with empty directory name", dirName != "")
if verb:
log.info(
blue(" - Resetting directory(") + yellow(dirName) + blue(",") +
red(resetOutDir) + blue(")"))
if os.path.isdir(dirName):
if resetOutDir:
filesInDir = os.listdir(dirName)
for fileNow in filesInDir:
os.system("rm -v " + dirName + "/" + fileNow)
else:
os.system("mkdir -vp " + dirName)
def genRndOpts(aSeed=0):
#rnd.seed(1982516+aSeed)
#rnd.seed(5264256+aSeed)
rndAr = [rnd.random() for i in range(15)]
if not "rndOptTypes" in glob.annz: glob.annz["rndOptTypes"] = "BDT"
rndOptTypes = glob.annz["rndOptTypes"]
isReg = glob.annz["doRegression"] and (not glob.annz["doBinnedCls"])
doANN = "ANN" in rndOptTypes
doBDT = "BDT" in rndOptTypes
doKNN = "KNN" in rndOptTypes
Assert("Must allow either \"ANN\", \"BDT\" or both in \"rndOptTypes\"",
(doANN or doBDT or doKNN))
if doANN and doBDT:
doANN = (rndAr[0] < 0.5)
doBDT = (not doANN)
if doANN:
mlmType = "ANN"
nLayer0 = int(floor(rndAr[1] * 5))
nLayer1 = int(floor(rndAr[2] * 10))
nLayer2 = int(floor(rndAr[3] * 5))
layerArc = "N+" + str(nLayer0) + ",N+" + str(nLayer1)
if (rndAr[1] < 0.3 and rndAr[4] < 0.3):
layerArc += ",N+" + str(nLayer2)
layerArc = layerArc.replace("N+0", "N")
neuronInput = "sum"
neuronType = "tanh" if (rndAr[5] < 0.5) else "sigmoid"
useReg = "True" if (rndAr[6] < 0.5) else "False"
resetStep = "100" if (rndAr[7] < 0.3) else ("250" if (
rndAr[7] < 0.65) else "500")
convTests = "-1" if (rndAr[8] < 0.3) else ("25" if
(rndAr[8] < 0.65) else "50")
testRate = "5" if (rndAr[9] < 0.3) else ("25" if
(rndAr[9] < 0.65) else "50")
trainMethod = "BFGS" if (rndAr[10] < 0.7 and isReg) else "BP"
# sometimes BFGS crashes for classification ... ?!
RandomSeed = ":RandomSeed=" + str(int(floor(rndAr[11] * 100000)))
opt = ":HiddenLayers="+layerArc+":NeuronType="+neuronType+":NeuronInputType="+neuronInput \
+":TrainingMethod="+trainMethod+":TestRate="+testRate+":NCycles=5000" \
+":UseRegulator="+useReg+":ConvergenceTests="+convTests+":ConvergenceImprove=1e-30" \
+":SamplingTraining=False:SamplingTesting=False"+":ResetStep="+resetStep+RandomSeed
elif doBDT:
mlmType = "BDT"
nTreeFact = 3 if (rndAr[1] < 0.2) else 1
nTreesAdd = int(floor(rndAr[2] * 300 / 10.) * 10) * nTreeFact
nTrees = ":NTrees=" + str(int(250 + max(0, min(nTreesAdd, 800))))
boostType = ":BoostType="
if (rndAr[3] < 0.4): boostType += "Bagging"
elif (rndAr[3] < 0.8): boostType += "AdaBoost"
else:
if isReg: boostType += "AdaBoostR2" # only for regression
else: boostType += "Grad" # only for calssification
nEventsMin = ":nEventsMin=" + str(3 + int(floor(rndAr[4] * 57)))
nCuts = ":nCuts=" + str(10 + int(floor(rndAr[5] * 30 / 5.) * 5)) if (
rndAr[6] < 0.3) else ""
opt = nTrees + boostType #+nEventsMin+nCuts;
elif doKNN:
mlmType = "KNN"
nKnnDef = 20
nKnnMod = int(floor(rndAr[1] * 20 - 10))
nKnn = str(nKnnDef + nKnnMod)
balDepthDef = 6
balDepthMod = int(floor(rndAr[2] * 6 - 3))
balDepth = str(balDepthDef + balDepthMod)
#if rndAr[3] < 0.5: kernel = ":Kernel=Gaus"
#else: kernel = ":Kernel=Poln"
kernel = ":Kernel=Gaus"
opt = ":nkNN=" + nKnn + ":BalanceDepth=" + balDepth + kernel
rndAr = [rnd.random() for i in range(2)]
varTrans = ""
if (rndAr[0] < 0.9): varTrans += "N"
if (rndAr[1] < 0.5): varTrans += "P"
varTrans = varTrans.replace("NP", "N,P")
if (varTrans != ""): varTrans = ":VarTransform=" + varTrans
opt = "ANNZ_MLM=" + mlmType + opt + varTrans
return opt