def convertTree(config, treeName, category):
""" Wrapper for the functionality of preprocessing.dataset """
logging.info("Starting conversion")
checkNcreateFolder(config.outputFolder)
datasetName = config.outputPrefix + "_" + config.sampleName + "_" + config.categories[
category].name
dataset = Dataset(datasetName, config.outputFolder, treeName)
logging.info("Setting sample selection: %s", config.sampleSelection)
dataset.sampleSelection = config.sampleSelection
logging.info("Setting category selection: %s",
config.categories[category].selection)
dataset.selection = config.categories[category].selection
if config.excludeBranches is not None:
dataset.ignoreBranches = config.excludeBranches
logging.info("Setting files")
dataset.addFiles(config.files)
logging.info("Setting output branches")
dataset.setOutputBranches(config.outputVariables)
logging.debug("Setting indexing branches: %s", config.indexVariables)
dataset.outputIndex = config.indexVariables
if config.addRatio:
dataset.setSF(config.sampleSF, "sampleRatio")
logging.info("Starting processing dataset")
dataset.process(config.maxEvents)
logging.info("Finished processing")
def convertTree(inputs, outFolder, name, treeName, indexVars):
logging.info("Starting conversion")
checkNcreateFolder(outFolder)
dataset = Dataset(name, outFolder, treeName)
files = []
for _input in inputs:
with open(_input, "r") as f:
data = f.read()
for line in data.split("\n"):
if ".root" in line:
files.append(line)
logging.info("Setting files")
dataset.addFiles(files)
logging.info("Setting output branches")
dataset.setOutputBranches("*")
logging.debug("Setting indexing branches: %s", indexVars)
dataset.outputIndex = indexVars
logging.info("Starting processing dataset")
dataset.process(999999999999999999999)
logging.info("Finished processing")
def createLookup(model, inputFile, outputFolder):
checkNcreateFolder(outputFolder, onlyFolder=True)
dfPrediction, _, _ = processData(model, inputFile)
inputFileName = inputFile.split("/")[-1]
inputFileName = inputFileName.split(".")[0]
writeLookupTable(dfPrediction, outputFolder, inputFileName)
def process(args, styleConfig):
inputDFs = getDataframes([
filename for filename in args.input if not filename.startswith("merge")
])
vars2Process = generateVariableList(inputDFs[0], args.plotVars,
args.excludeVars)
if len([
filename for filename in args.input if filename.startswith("merge")
]) == 1:
mergedName, mergedDF, weightedDFs = mergeDatasets(args, vars2Process)
#print(mergedDF)
inputDFs = inputDFs + [mergedDF]
if args.inputNames is not None:
args.inputNames = args.inputNames + [mergedName]
if args.inputNames is not None:
assert len(args.inputNames) == len(inputDFs)
if args.plotCorr:
logging.info("Will do correlation plots")
for iDF, inputDF in enumerate(inputDFs):
logging.info("Processing file %s", inputDFs)
outpath = args.output.split("/")
if len(
outpath
) > 1: #In this case we want to insert the folder for the dataframe
folders = outpath[0:-1]
folders.append("DF-" +
str(iDF) if args.inputNames is None else args.
inputNames[iDF])
thisOutput = "/".join(folders + [outpath[-1]])
thisOutput += "_corr2D_"
checkNcreateFolder(thisOutput)
else:
thisOutput = args.output + "_corr2D_" + (str(
iDF) if args.inputNames is None else args.inputNames[iDF])
getCorrealtions(styleConfig, inputDF, thisOutput, vars2Process,
args.transform)
if args.plotDist:
logging.info("Will plot distributions")
thisOutput = args.output + "_dist_"
checkNcreateFolder(thisOutput)
getDistributions(styleConfig, inputDFs, thisOutput, vars2Process,
args.inputNames, args.normalized, args.transform,
args.lumi, args.CMSString, args.colors, args.catlabel,
args.yScale)
def convertTreeMulti(config, treeName, category):
logging.info("Starting conversion using multi method")
checkNcreateFolder(config.outputFolder)
#For multi mode, we generate a dataset per sample. In the loop the output is disabled and in the end the
#dataframs of the 1:: samples will be added to the first and saved
eventsLeft = config.maxEvents
dfs = []
baseDataset = None
for iSample, sample in enumerate(config.samples):
logging.info("Processing sample %s", sample)
if iSample == 0:
datasetName = config.outputPrefix + "_" + config.sampleName + "_" + config.categories[
category].name
else:
datasetName = config.outputPrefix + "_" + config.sampleInfo[
sample].name + "_" + config.categories[category].name
dataset = Dataset(datasetName, config.outputFolder, treeName)
logging.info("Setting sample selection: %s",
config.sampleInfo[sample].selection)
dataset.sampleSelection = config.sampleInfo[sample].selection
logging.info("Setting category selection: %s",
config.categories[category].selection)
dataset.selection = config.categories[category].selection
if config.excludeBranches is not None:
dataset.ignoreBranches = config.excludeBranches
logging.info("Setting files")
dataset.addFiles(config.sampleInfo[sample].files)
logging.info("Setting output branches")
dataset.setOutputBranches(config.outputVariables)
logging.debug("Setting indexing branches: %s", config.indexVariables)
dataset.outputIndex = config.indexVariables
if config.addRatio:
dataset.setSF(config.sampleInfo[sample].addSF, "sampleRatio")
logging.info("Starting processing dataset")
thisSampleDF = dataset.process(eventsLeft, skipOutput=True)
eventsLeft -= len(thisSampleDF)
dfs.append(thisSampleDF)
if iSample == 0:
baseDataset = copy(dataset)
baseDataset.makeOutput(pd.concat(dfs))
logging.info("Finished processing")
def trainAutoencoder(config, useDevice, batch=False):
logging.debug("Output folder")
checkNcreateFolder(config.output, onlyFolder=True)
logging.debug("Copying used config to outputfolder")
shutil.copy2(config.path, config.output + "/usedConfig.cfg")
logging.info("Initializing samples and data")
allSample, data = initialize(config)
logging.debug("Getting activations")
thisEncoderActivation, thisDecoderActivation = buildActivations(config)
logging.debug("Encoder: %s", thisEncoderActivation)
logging.debug("Decoder: %s", thisDecoderActivation)
logging.info("Initializing autoencoder")
thisAutoencoder = Autoencoder(identifier=config.net.name,
inputDim=config.net.inputDimention,
encoderDim=config.encoder.dimention,
hiddenLayerDim=[
config.hiddenLayers[i].dimention
for i in range(config.nHiddenLayers)
],
weightDecay=config.net.useWeightDecay,
robust=config.net.robustAutoencoder,
encoderActivation=thisEncoderActivation,
decoderActivation=thisDecoderActivation,
loss=config.net.loss,
metric=['mae', "msle", "acc"],
batchSize=config.net.batchSize)
logging.info("Setting optimizer")
logging.debug("In config: %s", config.net.optimizer)
thisAutoencoder.optimizer = config.net.optimizer
#thisAutoencoder.setOptimizer(optimizerName=config.net.optimizer)
logging.info("Building model")
thisAutoencoder.buildModel()
logging.info("Compiling model")
thisAutoencoder.compileModel()
trainData = data.getTrainData()
# print(data.trainVariables)
# print(data.trainDF[data.trainVariables])
# #print(data.untransfromedDF[data.trainVariables[1]])
# print(trainData)
# input("Press ret")
testData = data.getTestData()
trainWeights = data.trainTrainingWeights
testWeights = data.testTrainingWeights
logging.info("Fitting model")
thisAutoencoder.network.summary()
if not batch:
input("Press ret")
thisAutoencoder.trainModel(trainData,
trainWeights,
config.output,
epochs=config.net.trainEpochs,
valSplit=config.net.validationSplit,
thisDevice="",
earlyStopping=(args.stopEarly
or config.net.doEarlyStopping),
patience=config.net.StoppingPatience)
logging.info("Evaluation....")
predictedData = thisAutoencoder.evalModel(testData, testWeights,
data.trainVariables,
config.output, True, True)
logging.info("Getting reco error for loss")
reconstMetric, reconstErrTest = thisAutoencoder.getReconstructionErr(
testData)
reconstMetric, reconstErrTrain = thisAutoencoder.getReconstructionErr(
trainData)
make1DHistoPlot([reconstErrTest, reconstErrTrain],
None,
"{0}/{1}_{2}".format(config.output, "TrainingReconst",
reconstMetric),
20, (0, 2),
"Loss function", ["Test Sample", "Training Sample"],
normalized=True)
logging.info("Saving testData and weights")
data2Pickle = {
"variables": config.trainingVariables,
"testInputData": testData,
"testWeights": testWeights,
"testPredictionData": predictedData
}
with open("{0}/testDataArrays.pkl".format(config.output),
"wb") as pickleOut:
pickle.dump(data2Pickle, pickleOut)
thisAutoencoder.saveModel(config.output, data.transformations)
def main(args):
checkNcreateFolder(args.output, onlyFolder=True)
evalData = {}
for iData, pickledData in enumerate(args.input):
with open(pickledData, "rb") as f:
evalData[args.inputID[iData]] = pickle.load(f)
assert evalData[args.inputID[0]]["variables"] == evalData[
args.inputID[1]]["variables"]
variables = evalData[args.inputID[0]]["variables"]
datasets = evalData[args.inputID[0]]["datasets"]
if args.skipDataset is not None:
for ds in args.skipDataset:
datasets.remove(ds)
inputData = {}
predictionData = {}
for name in args.inputID:
inputData[name] = {}
predictionData[name] = {}
for iVar, var in enumerate(variables):
logging.debug("Adding array for output %s, variable %s", name, var)
dataList = [
evalData[name]["inputData"][i][:, iVar]
for i in range(len(evalData[name]["inputData"]))
]
dataListPred = [
evalData[name]["predictionData"][i][:, iVar]
for i in range(len(evalData[name]["predictionData"]))
]
inputData[name][var] = {}
predictionData[name][var] = {}
for iDataset, dataset in enumerate(datasets):
inputData[name][var][dataset] = dataList[iDataset]
logging.debug("Added input data for %s with len %s", dataset,
len(inputData[name][var][dataset]))
predictionData[name][var][dataset] = dataListPred[iDataset]
logging.debug("Added prediction data for %s with len %s",
dataset, len(inputData[name][var][dataset]))
if args.transform:
transtransforms = {}
for iName, name in enumerate(args.inputID):
with open(args.useTransform, "r") as f:
transtransforms[name] = json.load(f)
for iVar, var in enumerate(variables):
for iDataset, dataset in enumerate(datasets):
inputData[name][var][dataset] = transform(
"Gauss", inputData[name][var][dataset], [var],
transtransforms[name])
predictionData[name][var][dataset] = transform(
"Gauss", predictionData[name][var][dataset], [var],
transtransforms[name])
logging.info("Starting plotting")
for var in variables:
logging.info("Plotting var %s", var)
for dataset in datasets:
dataSetCompInput = []
dataSetCompPred = []
dataSetCompAll = []
legendInput = []
legendPrediction = []
legendAll = []
for name in args.inputID:
dataSetCompInput.append(inputData[name][var][dataset])
dataSetCompPred.append(predictionData[name][var][dataset])
dataSetCompAll.append(inputData[name][var][dataset])
dataSetCompAll.append(predictionData[name][var][dataset])
thisTextInput = "%s Input data (%s)" % (name, dataset)
thisTextPrediction = "%s Predicted data (%s)" % (name, dataset)
legendInput.append(thisTextInput)
legendPrediction.append(thisTextPrediction)
legendAll.append(thisTextInput)
legendAll.append(thisTextPrediction)
make1DHistoPlot(dataSetCompInput,
None,
"{0}/comp_input_{1}_{2}".format(
args.output, var, dataset),
nBins=40,
binRange=(-4, 4),
varAxisName=var,
legendEntries=legendInput,
normalized=True)
make1DHistoPlot(dataSetCompPred,
None,
"{0}/comp_prediction_{1}_{2}".format(
args.output, var, dataset),
nBins=40,
binRange=(-4, 4),
varAxisName=var,
legendEntries=legendPrediction,
normalized=True)
make1DHistoPlot(dataSetCompAll,
None,
"{0}/comp_all_{1}_{2}".format(
args.output, var, dataset),
nBins=40,
binRange=(-4, 4),
varAxisName=var,
legendEntries=legendAll,
normalized=True)
def main(config):
checkNcreateFolder(config.output)
for dataset in config.runDatasets:
logging.info("Processing dataset %s", dataset)
finalDF = processDataset(config, dataset)
createlookup_dnn.writeLookupTable(finalDF, config.output, dataset)
thisCatLabel = plotSettings["catLabel"]
else:
thisCatLabel = None
if "yScale" in plotSettings.keys():
thisYScale = plotSettings["yScale"]
else:
thisYScale = 1.25
#thisCMSString = None
styleConfig = StyleConfig(plotSettings["style"])
inputDFs = checkInputData.getDataframes([filename for filename in plotSettings["inputs"]])
checkNcreateFolder(os.path.expanduser(plotSettings["output"]))
checkInputData.getDistributions(
styleConfig,
inputDFs,
os.path.expanduser(plotSettings["output"]),
["MEM"],
plotSettings["Legend"],
plotSettings["normalized"],
False,
plotSettings["lumi"],
thisCMSString,
theseColors,
thisCatLabel,
thisYScale,
def trainDNN(config, batch=False, addMetrics=["MEM"]):
logging.debug("Output folder")
checkNcreateFolder(config.output, onlyFolder=True)
logging.debug("Copying used config to outputfolder")
shutil.copy2(config.path, config.output + "/usedConfig.cfg")
logging.info("Initializing samples and data")
allSample, data = initialize(config, incGenWeights=config.includeGenWeight)
logging.info("Initializing DNN")
thisDNN = DNN(identifier=config.net.name,
inputDim=config.net.inputDimention,
layerDims=config.net.layerDimentions,
weightDecay=config.net.useWeightDecay,
weightDecayLambda=config.net.weightDecayLambda,
activation=config.net.activation,
outputActivation=config.net.outputActivation,
loss=config.net.loss,
metric=["acc"],
batchSize=config.net.batchSize)
logging.info("Setting optimizer")
logging.debug("In config: %s", config.net.optimizer)
thisDNN.optimizer = config.net.optimizer
logging.info("Building model")
if config.net.loss == "binary_crossentropy":
thisDNN.buildModel(nClasses=1,
dropoutAll=config.net.dropoutAll,
dropoutOutput=config.net.dropoutOutput,
dropoutPercent=config.net.dropoutPercent)
else:
thisDNN.buildModel(nClasses=len(data.outputClasses),
dropoutAll=config.net.dropoutAll,
dropoutOutput=config.net.dropoutOutput,
dropoutPercent=config.net.dropoutPercent)
logging.info("Compiling model")
thisDNN.compileModel()
thisDNN.network.summary(print_fn=logging.warning)
if not batch:
input("Press ret")
trainData = data.getTrainData()
trainLabels = data.trainLabels
trainWeights = data.trainTrainingWeights
logging.info("Training DNN")
thisDNN.trainModel(trainData,
trainLabels,
trainWeights,
config.output,
epochs=config.net.trainEpochs,
valSplit=config.net.validationSplit,
earlyStopping=config.net.doEarlyStopping,
patience=config.net.StoppingPatience)
testData = data.getTestData()
testLabels = data.testLabels
testWeights = data.testTrainingWeights
#TODO: Make this configurable
ROCMetrics = []
if (len(addMetrics) == 1 and addMetrics[0] == ""):
addMetrics = []
for metric in addMetrics:
ROCMetrics.append(
(metric, data.getTestData(asMatrix=False)[metric].values))
logging.info("Model evaluation")
thisDNN.evalModel(testData,
testWeights,
testLabels,
trainData,
trainWeights,
trainLabels,
config.trainingVariables,
config.output,
data.outputClasses,
plotMetics=True,
saveData=True,
addROCMetrics=ROCMetrics,
forceColors=config.forceColors)
logging.info("Saving model")
thisDNN.saveModel(config.output, data.transformations)