def GetCorrelationMatrixInJSON(className, varNames, matrix):
m = ROOT.TMatrixD(len(matrix), len(matrix))
for i in xrange(len(matrix)):
for j in xrange(len(matrix)):
m[i][j] = matrix[i][j]
th2 = ROOT.TH2D(m)
th2.SetTitle("Correlation matrix (" + className + ")")
for i in xrange(len(varNames)):
th2.GetXaxis().SetBinLabel(i + 1, varNames[i])
th2.GetYaxis().SetBinLabel(i + 1, varNames[i])
th2.Scale(100.0)
for i in xrange(len(matrix)):
for j in xrange(len(matrix)):
th2.SetBinContent(i + 1, j + 1,
int(th2.GetBinContent(i + 1, j + 1)))
th2.SetStats(0)
th2.SetMarkerSize(1.5)
th2.SetMarkerColor(0)
labelSize = 0.040
th2.GetXaxis().SetLabelSize(labelSize)
th2.GetYaxis().SetLabelSize(labelSize)
th2.LabelsOption("d")
th2.SetLabelOffset(0.011)
th2.SetMinimum(-100.0)
th2.SetMaximum(+100.0)
dat = TBufferJSON.ConvertToJSON(th2)
return str(dat).replace("\n", "")
def GetCorrelationMatrixInJSON(className, varNames, matrix):
m = ROOT.TMatrixD(len(matrix), len(matrix))
for i in xrange(len(matrix)):
for j in xrange(len(matrix)):
m[i][j] = matrix[i][j]
th2 = ROOT.TH2D(m)
th2.SetTitle("Correlation matrix ("+className+")")
for i in xrange(len(varNames)):
th2.GetXaxis().SetBinLabel(i+1, varNames[i])
th2.GetYaxis().SetBinLabel(i+1, varNames[i])
th2.Scale(100.0)
for i in xrange(len(matrix)):
for j in xrange(len(matrix)):
th2.SetBinContent(i+1, j+1, int(th2.GetBinContent(i+1, j+1)))
th2.SetStats(0)
th2.SetMarkerSize(1.5)
th2.SetMarkerColor(0)
labelSize = 0.040
th2.GetXaxis().SetLabelSize(labelSize)
th2.GetYaxis().SetLabelSize(labelSize)
th2.LabelsOption("d")
th2.SetLabelOffset(0.011)
th2.SetMinimum(-100.0)
th2.SetMaximum(+100.0)
dat = TBufferJSON.ConvertToJSON(th2)
return str(dat).replace("\n", "")
def __transformVariableMeanSpecificContent(self, headerMatch, startIndex, maxlen):
count = 0
table = [[]]
for j in range(1, 6):
table[0].append(headerMatch.group(j))
for l in xrange(1, maxlen):
nextline = self.lines[startIndex + l]
if self.__isEmpty(nextline):
count += 1
continue
VariableMean = re.match(r"\s*:\s*([\w\d]+)\s*:\s*(-?\d*\.?\d*)\s*(-?\d*\.?\d*)\s*\[\s*(-?\d*\.?\d*)\s*(-?\d*\.?\d*)\s*\]", nextline, re.I)
if VariableMean:
count += 1
tmp = []
for j in range(1, 6):
tmp.append(VariableMean.group(j))
table.append(tmp)
else:
break
rstr = "<table class='tmva_output_varmeanrms'>"
for i in xrange(len(table)):
rstr += "<tr>"
for j in xrange(len(table[i])):
rstr += "<td>" + str(table[i][j])
rstr += "<td class='tmva_output_hidden_td'></td>"
rstr += "</tr>"
rstr += "</table>"
return (count, rstr)
def CreateWeightHist(net, selectedLayers):
firstLayer = int(selectedLayers.split("->")[0])
weights = net["layers"][firstLayer]["Weights"]
n1 = int(weights["row"])
n2 = int(weights["cols"])
m = ROOT.TMatrixD(n1, n2+1)
vec = weights["data"]
for i in xrange(n1):
for j in xrange(n2):
m[i][j] = vec[j+n2*i]
bvec = net["layers"][firstLayer]["Biases"]["data"]
if n1!=len(bvec):
print("Something wrong.. Number of bias weights not equal with the neuron number ("+str(n1)+"!="+str(len(bvec))+")")
return
for i in xrange(n1):
m[i][n2] = bvec[i]
th2 = ROOT.TH2D(m)
th2.SetTitle("Weight map for DNN")
for i in xrange(n2):
th2.GetXaxis().SetBinLabel(i + 1, str(i))
th2.GetXaxis().SetBinLabel(n2+1, "B")
for i in xrange(n1):
th2.GetYaxis().SetBinLabel(i + 1, str(i))
th2.GetXaxis().SetTitle("Layer: "+str(firstLayer))
th2.GetYaxis().SetTitle("Layer: "+str(firstLayer+1))
th2.SetStats(0)
th2.SetMarkerSize(1.5)
th2.SetMarkerColor(0)
labelSize = 0.040
th2.GetXaxis().SetLabelSize(labelSize)
th2.GetYaxis().SetLabelSize(labelSize)
th2.LabelsOption("d")
th2.SetLabelOffset(0.011)
clear_output()
JPyInterface.JsDraw.Draw(th2, 'drawDNNMap')
def __transformDatasetSpecificContent(self, firstLine, startIndex, maxlen):
tmp_str = ""
count = 0
for l in xrange(1, maxlen):
nextline = self.lines[startIndex+l]
if self.__isEmpty(nextline):
count += 1
continue
DatasetName = re.match(r".*(\[.*\])\s*:\s*(.*)", nextline)
if DatasetName:
count += 1
tmp_str += "<tr>"+self.__processGroupContentLine(DatasetName.group(2))
tmp_str += "<td class='tmva_output_hidden_td'></td></tr>"
else:
break
DatasetName = re.match(r".*(\[.*\])\s*:\s*(.*)", firstLine)
self.__lastDataSetName = DatasetName.group(1).replace("[", "").replace("]", "")
tbodyclass = ""
if count > 0:
tbodyclass = " class='tmva_output_tbody_multiple_row'"
rstr = "<table class='tmva_output_dataset'><tbody"+tbodyclass+">"
rstr += "<tr><td rowspan='"+str(count+1)+"'>Dataset: "+self.__lastDataSetName+"</td>"
rstr += self.__processGroupContentLine(DatasetName.group(2)) + "<td class='tmva_output_hidden_td'></td></tr>"
rstr += tmp_str
rstr += "</tbody></table>"
return (count, rstr)
def DrawDNNWeights(fac, datasetName, methodName="DNN"):
m = GetMethodObject(fac, datasetName, methodName)
if m == None:
return None
try:
net = GetDeepNetwork(str(m.GetWeightFileName()), True)
except AttributeError:
print(
"STANDARD architecture not supported! If you want to use this function you must use CPU or GPU architecture"
)
numOfLayers = len(net["layers"])
options = []
vals = []
for layer in xrange(numOfLayers):
options.append(str(layer) + "->" + str(layer + 1))
vals.append(layer)
selectLayer = widgets.Dropdown(options=options,
value=options[0],
description='Layer')
def drawWrapper(e):
CreateWeightHist(net, selectLayer.value)
pass
button = widgets.Button(description="Draw",
font_weight="bold",
font_size="16")
button.on_click(drawWrapper)
box = widgets.HBox([selectLayer, button])
display(box)
def ChangeCallOriginal__init__(*args, **kwargs):
hasColor = False
args = list(args)
for arg_idx in xrange(len(args)):
# basestring==(str, unicode) in Python2, which translates to str in Python3
if sys.version_info >= (3, 0):
is_string = isinstance(args[arg_idx], str)
else:
is_string = isinstance(args[arg_idx], basestring)
if is_string and args[arg_idx].find(":") != -1:
if args[arg_idx].find("Color") != -1:
hasColor = True
if args[arg_idx].find("!Color") == -1:
args[arg_idx] = args[arg_idx].replace("Color", "!Color")
else:
kwargs["Color"] = False
args = tuple(args)
if not hasColor:
kwargs["Color"] = False
try:
args, kwargs = JPyInterface.functions.ConvertSpecKwargsToArgs(
["JobName", "TargetFile"], *args, **kwargs)
except AttributeError:
try:
args, kwargs = JPyInterface.functions.ConvertSpecKwargsToArgs(
["JobName"], *args, **kwargs)
except AttributeError:
raise AttributeError
originalFunction, args = JPyInterface.functions.ProcessParameters(
3, *args, **kwargs)
return originalFunction(*args)
def DrawDNNWeights(fac, datasetName, methodName="DNN"):
m = GetMethodObject(fac, datasetName, methodName)
if m == None:
return None
try:
net = GetDeepNetwork(str(m.GetWeightFileName()), True)
except AttributeError:
print("STANDARD architecture not supported! If you want to use this function you must use CPU or GPU architecture")
numOfLayers = len(net["layers"])
options = []
vals=[]
for layer in xrange(numOfLayers):
options.append(str(layer)+"->"+str(layer+1))
vals.append(layer)
selectLayer=widgets.Dropdown(
options=options,
value=options[0],
description='Layer'
)
def drawWrapper(e):
CreateWeightHist(net, selectLayer.value)
pass
button = widgets.Button(description="Draw", font_weight="bold", font_size="16")
button.on_click(drawWrapper)
box = widgets.HBox([selectLayer, button])
display(box)
def ChangeCallOriginal__init__(*args, **kwargs):
hasColor = False
args = list(args)
for arg_idx in xrange(len(args)):
# basestring==(str, unicode) in Python2, which translates to str in Python3
if sys.version_info >= (3, 0):
is_string = isinstance(args[arg_idx], str)
else:
is_string = isinstance(args[arg_idx], basestring)
if is_string and args[arg_idx].find(":")!=-1:
if args[arg_idx].find("Color")!=-1:
hasColor = True
if args[arg_idx].find("!Color")==-1:
args[arg_idx] = args[arg_idx].replace("Color", "!Color")
else:
kwargs["Color"] = False
args = tuple(args)
if not hasColor:
kwargs["Color"] = False
try:
args, kwargs = JPyInterface.functions.ConvertSpecKwargsToArgs(["JobName", "TargetFile"], *args, **kwargs)
except AttributeError:
try:
args, kwargs = JPyInterface.functions.ConvertSpecKwargsToArgs(["JobName"], *args, **kwargs)
except AttributeError:
raise AttributeError
originalFunction, args = JPyInterface.functions.ProcessParameters(3, *args, **kwargs)
return originalFunction(*args)
def __transformNumEvtSpecificContent(self, firstLine, startIndex, maxlen):
tmp_str = ""
count = 0
tmpmap = {}
for l in xrange(1, maxlen):
nextline = self.lines[startIndex + l]
if self.__isEmpty(nextline):
count += 1
continue
NumberOfEvents = re.match(
r"\s+:\s*\w+\s*-\s*-\s*((training\sevents)|(testing\sevents)|(training\sand\stesting\sevents))\s*:\s*\d+",
nextline)
if NumberOfEvents:
lc = re.findall(r"\w+", nextline)
t = ""
for i in range(1, len(lc) - 1):
t += lc[i] + " "
t = t[:-1]
if lc[0] not in tmpmap:
tmpmap[lc[0]] = []
count += 1
tmpmap[lc[0]].append({"name": t, "value": lc[len(lc) - 1]})
else:
break
rstr = "<table class='tmva_output_traintestevents'>"
rstr += "<tr><td colspan='3'><center><b>" + firstLine + "</b></center></td></tr>"
for key in tmpmap:
rstr += "<tr>"
rstr += "<td rowspan='" + str(len(
tmpmap[key])) + "'>" + key + "</td>"
rstr += "<td>" + tmpmap[key][0]["name"] + "</td><td>" + tmpmap[
key][0]["value"] + "</td>"
rstr += "<td class='tmva_output_hidden_td'></td>"
rstr += "</tr>"
for i in xrange(1, len(tmpmap[key])):
rstr += "<tr><td>" + tmpmap[key][i][
"name"] + "</td><td>" + tmpmap[key][i]["value"] + "</td>"
rstr += "<td class='tmva_output_hidden_td'></td></tr>"
rstr += tmp_str
rstr += "</table>"
return (count, rstr)
def CreateWeightHist(net, selectedLayers):
firstLayer = int(selectedLayers.split("->")[0])
weights = net["layers"][firstLayer]["Weights"]
n1 = int(weights["row"])
n2 = int(weights["cols"])
m = ROOT.TMatrixD(n1, n2 + 1)
vec = weights["data"]
for i in xrange(n1):
for j in xrange(n2):
m[i][j] = vec[j + n2 * i]
bvec = net["layers"][firstLayer]["Biases"]["data"]
if n1 != len(bvec):
print(
"Something wrong.. Number of bias weights not equal with the neuron number ("
+ str(n1) + "!=" + str(len(bvec)) + ")")
return
for i in xrange(n1):
m[i][n2] = bvec[i]
th2 = ROOT.TH2D(m)
th2.SetTitle("Weight map for DNN")
for i in xrange(n2):
th2.GetXaxis().SetBinLabel(i + 1, str(i))
th2.GetXaxis().SetBinLabel(n2 + 1, "B")
for i in xrange(n1):
th2.GetYaxis().SetBinLabel(i + 1, str(i))
th2.GetXaxis().SetTitle("Layer: " + str(firstLayer))
th2.GetYaxis().SetTitle("Layer: " + str(firstLayer + 1))
th2.SetStats(0)
th2.SetMarkerSize(1.5)
th2.SetMarkerColor(0)
labelSize = 0.040
th2.GetXaxis().SetLabelSize(labelSize)
th2.GetYaxis().SetLabelSize(labelSize)
th2.LabelsOption("d")
th2.SetLabelOffset(0.011)
clear_output()
JPyInterface.JsDraw.Draw(th2, 'drawDNNMap')
def __transformNumEvtSpecificContent(self, firstLine, startIndex, maxlen):
tmp_str = ""
count = 0
tmpmap = {}
for l in xrange(1, maxlen):
nextline = self.lines[startIndex+l]
if self.__isEmpty(nextline):
count += 1
continue
NumberOfEvents = re.match(r"\s+:\s*\w+\s*-\s*-\s*((training\sevents)|(testing\sevents)|(training\sand\stesting\sevents))\s*:\s*\d+", nextline)
if NumberOfEvents:
lc = re.findall(r"\w+", nextline)
t = ""
for i in range(1, len(lc) - 1):
t += lc[i] + " "
t = t[:-1]
if lc[0] not in tmpmap:
tmpmap[lc[0]] = []
count += 1
tmpmap[lc[0]].append({"name": t, "value": lc[len(lc) - 1]})
else:
break
rstr = "<table class='tmva_output_traintestevents'>"
rstr += "<tr><td colspan='3'><center><b>"+firstLine+"</b></center></td></tr>"
for key in tmpmap:
rstr += "<tr>"
rstr += "<td rowspan='"+str(len(tmpmap[key]))+"'>"+key+"</td>"
rstr += "<td>"+tmpmap[key][0]["name"]+"</td><td>"+tmpmap[key][0]["value"]+"</td>"
rstr += "<td class='tmva_output_hidden_td'></td>"
rstr += "</tr>"
for i in xrange(1, len(tmpmap[key])):
rstr += "<tr><td>"+tmpmap[key][i]["name"]+"</td><td>"+tmpmap[key][i]["value"]+"</td>"
rstr += "<td class='tmva_output_hidden_td'></td></tr>"
rstr += tmp_str
rstr += "</table>"
return (count, rstr)
def transform(self, output, error):
self.err = ""
if str(error).find(", time left:")==-1:
self.err = error
self.out = ""
self.lines = output.splitlines()
self.iterLines = iter(xrange(len(self.lines)))
for self.lineIndex in self.iterLines:
line = self.lines[self.lineIndex]
Header = re.match(r"^\s*-*\s*(<\w+>\s*)*\s*(\w+.*\s+)(\s+)(:)\s*(.*)", line, re.I)
EmptyHeader = re.match(r"\s*-*\s*(<\w+>\s*)+\s*:\s*(.*)", line, re.I)
NewGroup = re.match(r"^\s*:\s*$", line)
if Header:
if re.match(r"^\s*-+\s*(<\w+>\s*)*\s*(\w+.*\s+)(\s+)(:)\s*(.*)", line, re.I):
self.__currentType = "impHead"
else:
self.__currentType = Header.group(1)
self.addClassForOutputFlag(Header.group(5))
self.__currentHeaderName = Header.group(2)
self.__transformOneGroup(Header.group(5))
elif EmptyHeader:
self.__currentType = EmptyHeader.group(1)
self.addClassForOutputFlag(EmptyHeader.group(2))
self.__currentHeaderName = ""
self.__transformOneGroup(EmptyHeader.group(2))
elif NewGroup:
kw = 1
lines = []
while True:
if (self.lineIndex + kw) >=len(self.lines):
break
nextline = self.lines[self.lineIndex + kw]
kw += 1
if re.match(r"^\s*:\s*$", nextline):
Header = re.match(r"^\s*-*\s*(<\w+>\s*)*\s*(\w+.*\s+)(\s+)(:)\s*(.*)", self.lines[self.lineIndex+kw], re.I)
if Header:
self.iterLines.next()
break
self.iterLines.next()
if self.__isEmpty(nextline):
continue
lre = re.match(r"\s*:\s*(.*)", nextline)
if lre:
lines.append(lre.group(1))
else:
lines.append(nextline)
if len(lines)==0:
continue
if len(lines) > 1:
tbodyclass = " class='tmva_output_tbody_multiple_row'"
self.out += "<tbody"+tbodyclass+"><tr><td rowspan='"+str(len(lines))+"'><td>"+lines[0]+"</td></tr>"
for ii in xrange(1, len(lines)):
self.out += "<tr><td>"+lines[ii]+"</td></tr>"
else:
lre = re.match(r"\s*:\s*(.*)", line)
if lre:
self.out += "<tr><td>" + lre.group(1) + "</td></tr>"
else:
self.out += "<tr><td>" + line + "</td></tr>"
if len(self.out) < 1 and len(self.err) < 1:
return ("", "", "")
self.out = "<table class='tmva_output_table'>" + self.out + "</table>"
return (self.out, self.err, "html")
def __transformOneGroup(self, firstLine):
tmp_str = ""
processed_lines = 0
lineIter = iter(xrange(len(self.lines) - self.lineIndex))
for j in lineIter:
if j == 0:
nextline = firstLine
else:
nextline = self.lines[self.lineIndex + j]
Header = re.match(r"^\s*-*\s*(<\w+>\s*)*\s*(\w+.*\s+)(\s+)(:)\s*(.*)", nextline, re.I)
EmptyHeader = re.match(r"\s*-*\s*(<\w+>\s*)+\s*:\s*(.*)", nextline, re.I)
DatasetName = re.match(r".*(\[.*\])\s*:\s*(.*)", nextline)
NumEvents = re.match(r"(.*)(number\sof\straining\sand\stesting\sevents)", nextline, re.I)
CorrelationMatrixHeader = re.match(r".*\s*:?\s*(correlation\s*matrix)\s*\((\w+)\)\s*:\s*", nextline, re.I)
VariableMeanHeader = re.match(r".*\s*:?\s*(variable)\s*(mean)\s*(rms)\s*\[\s*(min)\s*(max)\s*\].*", nextline, re.I)
WelcomeHeader = re.match(r"^\s*:?\s*(.*ROOT\s*version:.*)", nextline, re.I)
if (Header == None and EmptyHeader ==None) or j == 0:
if j != 0:
processed_lines += 1
self.iterLines.next()
tmp_str += "<tr>"
if DatasetName or NumEvents or VariableMeanHeader:
if DatasetName:
func = self.__transformDatasetSpecificContent
fLine = nextline
elif NumEvents:
func = self.__transformNumEvtSpecificContent
fLine = NumEvents.group(2)
else:
func = self.__transformVariableMeanSpecificContent
fLine = VariableMeanHeader
count, tmp = func(fLine, self.lineIndex + j, len(self.lines) - self.lineIndex - j)
for x in xrange(count):
lineIter.next()
self.iterLines.next()
tmp_str += self.__processGroupContentLine(tmp)
elif CorrelationMatrixHeader:
self.iterLines.next()
lineIter.next()
ik = 1
matrixLines = []
while True:
self.iterLines.next()
lineIter.next()
ik += 1
if self.__isEmpty(self.lines[self.lineIndex + j + ik]):
break
ltmp = re.match(r"\s*:\s*(.*)", self.lines[self.lineIndex + j + ik]).group(1)
if ltmp.find(":") != -1:
matrixLines.append(ltmp.split(":"))
rmatch = "^\s*"
for ii in xrange(len(matrixLines)):
rmatch += r"(\+\d+\.?\d*|-\d+\.?\d*)\s*"
rmatch += "$"
matrix = []
varNames = []
for ii in xrange(len(matrixLines)):
varNames.append(matrixLines[ii][0])
ll = re.match(rmatch, matrixLines[ii][1])
mline = []
for jj in xrange(len(matrixLines)):
mline.append(float(ll.group(jj + 1)))
matrix.append(mline)
tmp_str += self.__correlationMatrix(CorrelationMatrixHeader.group(1),
CorrelationMatrixHeader.group(2), varNames, matrix)
elif WelcomeHeader:
kw = 0
while True:
nextline = self.lines[self.lineIndex + j + kw]
EndWelcome = re.match(r"\s*:?\s*_*\s*(TMVA\s*Version\s*.*)", nextline, re.I)
if EndWelcome or re.match(r"[\s_/|]*", nextline) == None:
break
kw += 1
self.iterLines.next()
lineIter.next()
tmp_str += "<td><b>" + WelcomeHeader.group(1) + "</b></td></tr>"
tmp_str += "<tr><td><img src='https://rawgit.com/root-mirror/root/master/tutorials/tmva/tmva_logo.svg' width='100%' /><br />"
tmp_str += "<center><b>" + EndWelcome.group(1) + "</b></center></td></tr>"
else:
lmatch = re.match(r"\s*(<\w+>\s*)*\s*:\s*(.*)", nextline)
if lmatch:
if lmatch.group(1):
self.__currentType = lmatch.group(1)
tmp_str += self.__processGroupContentLine(lmatch.group(2))
else:
tmp_str += self.__processGroupContentLine(nextline)
tmp_str += "</tr>"
else:
break
tbodyclass = ""
if processed_lines > 0:
tbodyclass = " class='tmva_output_tbody_multiple_row'"
extraHeaderClass=""
if self.__currentType=="impHead":
extraHeaderClass = " tmva_output_imphead"
self.out += "<tbody"+tbodyclass+"><tr><td rowspan='" + str(processed_lines + 1) + "' class='tmva_output_header"+extraHeaderClass+"'>" + self.__currentHeaderName + "</td>"
self.out += tmp_str + "</tbody>"
