def doMake(): hasLib = os.path.isdir(glob.libDirName) hasExe = os.path.isfile(glob.exeName) isClean = glob.pars["clean"] isMake = glob.pars["make"] or (not hasLib) or (not hasExe) if isClean or isMake: resetDir(glob.libDirName, isClean) if isMake: log.info( blue(" - Moving to ") + red(glob.libDirName) + blue(" and compiling ANNZ... ")) mkfl = os.path.join(glob.annzDir, 'Makefile') cmnd = "cd " + glob.libDirName + " ; make " + glob.makeOpt + " -f " + mkfl cmkdStatus = os.system(cmnd) Assert("compilation failed", (cmkdStatus == 0)) if os.path.isfile(glob.exeName): log.info( blue(" - Found ") + red(glob.exeName) + blue(" - compilation seems to have succeded... ")) # check that the executable exists before moving on and add the lib dir to LD_LIBRARY_PATH (needed on some systems) if not isClean: Assert("Did not find ANNZ executable (" + glob.exeName + ")", os.path.isfile(glob.exeName)) if not glob.libDirName in os.environ["LD_LIBRARY_PATH"]: if os.environ["LD_LIBRARY_PATH"] == "": os.environ["LD_LIBRARY_PATH"] = glob.libDirName else: os.environ["LD_LIBRARY_PATH"] = glob.libDirName + ":" + os.environ[ "LD_LIBRARY_PATH"] if glob.pars["onlyMake"]: exit(0)
def initROOT(): if "ROOTSYS" in os.environ and glob.useDefinedROOTSYS: log.info( blue(" - Found defined ") + green("ROOTSYS") + blue(" = \"") + yellow(os.environ["ROOTSYS"]) + blue("\". Setting glob.rootHome to match")) glob.rootHome = os.environ["ROOTSYS"] rootExeDir = glob.rootHome + "/bin/" rootExe = rootExeDir + "/root" rootHomeLib = glob.rootHome + "/lib/" Assert("Found rootHome = "+glob.rootHome+" which does not exist... Please set this in commonImports.py to the" \ +" ROOT installation directory !",os.path.isdir(glob.rootHome)) Assert("Did not find ROOT executable, expected as "+rootExe+" ..." \ +" Is ROOT properly installed ? Is glob.rootHome set to match ?",os.path.isfile(rootExe)) # add the bin directory of ROOT as first in the PATH, to make sure that the correct bin/root-config is used in the Makefile os.environ["PATH"] = rootExeDir + ":" + os.environ["PATH"] if not "ROOTSYS" in os.environ: os.environ["ROOTSYS"] = glob.rootHome log.info( blue(" - Setting ") + green("ROOTSYS") + blue(" = \"") + yellow(os.environ["ROOTSYS"]) + blue("\"")) elif glob.rootHome != os.environ["ROOTSYS"]: os.environ["ROOTSYS"] = glob.rootHome log.info( blue(" - Setting ") + green("ROOTSYS") + blue(" = \"") + yellow(os.environ["ROOTSYS"]) + blue("\"")) else: log.info( blue(" - Will use ") + green("ROOTSYS") + blue(" = \"") + yellow(os.environ["ROOTSYS"]) + blue("\"")) if not "LD_LIBRARY_PATH" in os.environ: os.environ["LD_LIBRARY_PATH"] = rootHomeLib log.info( blue(" - Setting ") + green("LD_LIBRARY_PATH") + blue(" = \"") + yellow(os.environ["LD_LIBRARY_PATH"]) + blue("\"")) elif not (rootHomeLib) in os.environ["LD_LIBRARY_PATH"]: os.environ["LD_LIBRARY_PATH"] = rootHomeLib + ":" + os.environ[ "LD_LIBRARY_PATH"] log.info( blue(" - Adding to ") + green("LD_LIBRARY_PATH") + blue(" \"") + yellow(rootHomeLib) + blue("\"")) else: log.info( blue(" - Found ") + green("LD_LIBRARY_PATH") + blue(" = \"") + yellow(os.environ["LD_LIBRARY_PATH"]) + blue("\"")) return
def runANNZ(): exitStatus = runOneANNZ() if exitStatus == 0: return # if ANNZ failed during training, try again with a different random seed and randomized MLM options # -------------------------------------------------------------------------------------------------- if glob.annz["doTrain"]: # set random seed if not already set setSeed = (not glob.annz.has_key("initSeedRnd")) if not setSeed: setSeed = (glob.annz["initSeedRnd"] < 1) if setSeed: glob.annz["initSeedRnd"] = 198876 # remove all userMLMopts so that a random set is generated glob.annz["userMLMopts"] = "" if glob.annz["doBinnedCls"]: for nRndOptNow in range(glob.annz["binCls_nTries"]): glob.annz.pop("userMLMopts_" + str(nRndOptNow), None) nTries = 0 for nTryNow in range(nTries): glob.annz["initSeedRnd"] += 1 log.warning( whtOnRed(" - runANNZ failed !!! Will try again (" + str(nTryNow + 1) + "/" + str(nTries) + ") with initSeedRnd = ") + yellowOnRed(str(glob.annz["initSeedRnd"]))) exitStatus = runOneANNZ() if exitStatus == 0: break Assert("runANNZ failed !!!", (exitStatus == 0)) return
def resetDir(dirName, resetOutDir, verb=True): Assert("Tried to resetDir with empty directory name", dirName != "") if verb: log.info( blue(" - Resetting directory(") + yellow(dirName) + blue(",") + red(resetOutDir) + blue(")")) if os.path.isdir(dirName): if resetOutDir: filesInDir = os.listdir(dirName) for fileNow in filesInDir: os.system("rm -v " + dirName + "/" + fileNow) else: os.system("mkdir -vp " + dirName)
def genRndOpts(aSeed=0): #rnd.seed(1982516+aSeed) #rnd.seed(5264256+aSeed) rndAr = [rnd.random() for i in range(15)] if not "rndOptTypes" in glob.annz: glob.annz["rndOptTypes"] = "BDT" rndOptTypes = glob.annz["rndOptTypes"] isReg = glob.annz["doRegression"] and (not glob.annz["doBinnedCls"]) doANN = "ANN" in rndOptTypes doBDT = "BDT" in rndOptTypes doKNN = "KNN" in rndOptTypes Assert("Must allow either \"ANN\", \"BDT\" or both in \"rndOptTypes\"", (doANN or doBDT or doKNN)) if doANN and doBDT: doANN = (rndAr[0] < 0.5) doBDT = (not doANN) if doANN: mlmType = "ANN" nLayer0 = int(floor(rndAr[1] * 5)) nLayer1 = int(floor(rndAr[2] * 10)) nLayer2 = int(floor(rndAr[3] * 5)) layerArc = "N+" + str(nLayer0) + ",N+" + str(nLayer1) if (rndAr[1] < 0.3 and rndAr[4] < 0.3): layerArc += ",N+" + str(nLayer2) layerArc = layerArc.replace("N+0", "N") neuronInput = "sum" neuronType = "tanh" if (rndAr[5] < 0.5) else "sigmoid" useReg = "True" if (rndAr[6] < 0.5) else "False" resetStep = "100" if (rndAr[7] < 0.3) else ("250" if ( rndAr[7] < 0.65) else "500") convTests = "-1" if (rndAr[8] < 0.3) else ("25" if (rndAr[8] < 0.65) else "50") testRate = "5" if (rndAr[9] < 0.3) else ("25" if (rndAr[9] < 0.65) else "50") trainMethod = "BFGS" if (rndAr[10] < 0.7 and isReg) else "BP" # sometimes BFGS crashes for classification ... ?! RandomSeed = ":RandomSeed=" + str(int(floor(rndAr[11] * 100000))) opt = ":HiddenLayers="+layerArc+":NeuronType="+neuronType+":NeuronInputType="+neuronInput \ +":TrainingMethod="+trainMethod+":TestRate="+testRate+":NCycles=5000" \ +":UseRegulator="+useReg+":ConvergenceTests="+convTests+":ConvergenceImprove=1e-30" \ +":SamplingTraining=False:SamplingTesting=False"+":ResetStep="+resetStep+RandomSeed elif doBDT: mlmType = "BDT" nTreeFact = 3 if (rndAr[1] < 0.2) else 1 nTreesAdd = int(floor(rndAr[2] * 300 / 10.) * 10) * nTreeFact nTrees = ":NTrees=" + str(int(250 + max(0, min(nTreesAdd, 800)))) boostType = ":BoostType=" if (rndAr[3] < 0.4): boostType += "Bagging" elif (rndAr[3] < 0.8): boostType += "AdaBoost" else: if isReg: boostType += "AdaBoostR2" # only for regression else: boostType += "Grad" # only for calssification nEventsMin = ":nEventsMin=" + str(3 + int(floor(rndAr[4] * 57))) nCuts = ":nCuts=" + str(10 + int(floor(rndAr[5] * 30 / 5.) * 5)) if ( rndAr[6] < 0.3) else "" opt = nTrees + boostType #+nEventsMin+nCuts; elif doKNN: mlmType = "KNN" nKnnDef = 20 nKnnMod = int(floor(rndAr[1] * 20 - 10)) nKnn = str(nKnnDef + nKnnMod) balDepthDef = 6 balDepthMod = int(floor(rndAr[2] * 6 - 3)) balDepth = str(balDepthDef + balDepthMod) #if rndAr[3] < 0.5: kernel = ":Kernel=Gaus" #else: kernel = ":Kernel=Poln" kernel = ":Kernel=Gaus" opt = ":nkNN=" + nKnn + ":BalanceDepth=" + balDepth + kernel rndAr = [rnd.random() for i in range(2)] varTrans = "" if (rndAr[0] < 0.9): varTrans += "N" if (rndAr[1] < 0.5): varTrans += "P" varTrans = varTrans.replace("NP", "N,P") if (varTrans != ""): varTrans = ":VarTransform=" + varTrans opt = "ANNZ_MLM=" + mlmType + opt + varTrans return opt