def showProgressBar(self, flag=True):
if flag and self.progress == None:
self.progress = ProgBar()
elif self.progress != None and not flag:
if self.progress.isActive():
self.progress.halt()
self.progress = None
def showProgressBar(self,flag=True):
if flag and self.progress==None:
self.progress = ProgBar()
elif self.progress!=None and not flag:
if self.progress.isActive():
self.progress.halt()
self.progress = None
def on_epoch_begin(self, epoch, logs={}):
self.target = self.params['samples']
self.progbar = ProgBar.Progbar(target=self.target,
newline_on_end=False)
self.seen = 0
self.progbar.last_run = 0
return
class PlotsBase:
variables = { }
cutflows = { }
def getVariables(self):
return PlotsBase.variables
def getVariables1D(self):
return [ v for v in PlotsBase.variables.values() if not v.is2D() ]
def getVariables2D(self):
return [ v for v in PlotsBase.variables.values() if v.is2D() ]
def addVariable(self,name,nbins,xmin,xmax,scut='l',uselog=True,binEdges=None):
assert name.isalnum()
assert not name in self.histogramList
if not name in PlotsBase.variables:
nb = nbins if binEdges else nbins/self.rebin
PlotsBase.variables[name] = Variable(name,nb,xmin,xmax,scut,uselog)
h1d = PlotsBase.variables[name].createHistogram(binEdges)
self.histogramList[name] = h1d
setattr(self,"h"+name,h1d)
def getCutFlows(self):
return PlotsBase.cutflows
def addCutFlow(self,labels,nameFlow="DefaultCutFlow"):
assert nameFlow.isalnum()
assert not nameFlow in self.cutflowList
if not nameFlow in PlotsBase.cutflows:
PlotsBase.cutflows[nameFlow] = CutFlow(nameFlow,labels)
h = PlotsBase.cutflows[nameFlow].createHistogram()
self.cutflowList[nameFlow] = h
setattr(self,"c"+nameFlow,h)
def addVariablePair(self,xname,nbinsx,xmin,xmax,yname,nbinsy,ymin,ymax,uselog=True,suffix=None):
varPair = VariablePair(xname,nbinsx/self.rebin,xmin,xmax,yname,nbinsy/self.rebin,ymin,ymax, \
uselog,suffix)
assert not varPair.name in self.histogramList
if not varPair.name in PlotsBase.variables:
PlotsBase.variables[varPair.name] = varPair
h2d = varPair.createHistogram()
self.histogramList[varPair.name] = h2d
setattr(self,"h"+varPair.name,h2d)
def __init__(self,name,preselection=None,elist=None,elistBase="./elists",rebin=1):
self.name = name
self.preselection = preselection
self.elist = elist
if elist!=None:
self.elist = elist.lower()[0]
self.elistBase = elistBase
assert os.path.isdir(elistBase)
self.timers = [ ]
for i in range(10):
self.timers.append(MyTimer())
self.writeElist = False
self.readElist = False
if self.preselection!=None and self.elist!=None:
self.preselName = self.preselection.__class__.__name__
if self.elist=="w" or self.elist=="a":
self.writeElist = True
elif self.elist=="r" or self.elist=="a":
self.readElist = True
if self.writeElist or self.readElist:
self.preselDirName = os.path.join(self.elistBase,self.preselName)
if not os.path.isdir(self.preselDirName):
os.mkdir(self.preselDirName,0744)
self.rebin = rebin
self.progress = None
def showProgressBar(self,flag=True):
if flag and self.progress==None:
self.progress = ProgBar()
elif self.progress!=None and not flag:
if self.progress.isActive():
self.progress.halt()
self.progress = None
def showTimers(self):
line = ""
for t in self.timers:
line += "{0:14.2f}".format(1000000*t.meanTime())
# line += " " + str(t.meanTime())
print line
def prepareElist(self,sample,subSampleName):
elist = None
elistFile = None
if self.readElist or self.writeElist:
dirName = os.path.join(self.preselDirName,sample.name)
if not os.path.isdir(dirName):
os.mkdir(dirName,0744)
elistFileName = os.path.join(dirName,subSampleName+"_elist.root")
if self.elist=="a":
self.writeElist = False
self.readElist = False
if not os.path.exists(elistFileName):
self.writeElist = True
else:
elistTime = os.path.getmtime(elistFileName)
if elistTime<=os.path.getmtime(self.preselection.sourcefile):
self.writeElist = True
if elistTime<=os.path.getmtime(sample.fullname(subSampleName)):
self.writeElist = True
self.readElist = not self.writeElist
if self.writeElist:
print "(Re)creating elist for ",sample.name,subSampleName
if self.writeElist:
print "Reading elist for ",sample.name,subSampleName
opt = "recreate" if self.writeElist else "read"
elistFile = ROOT.TFile(elistFileName,opt)
objarr = ROOT.TObjArray()
if self.writeElist:
objstr = ROOT.TObjString()
objstr.SetString(sample.name)
objarr.Add(objstr.Clone())
objstr.SetString(subSampleName)
objarr.Add(objstr.Clone())
objstr.SetString(str(sample.downscale))
objarr.Add(objstr.Clone())
objarr.Write("file",ROOT.TObject.kSingleKey)
elist = ROOT.TEventList("elist",self.preselName+" / "+sample.name+" / "+subSampleName)
else:
objarr = elistFile.Get("file")
assert objarr[0].GetString().Data()==sample.name
assert objarr[1].GetString().Data()==subSampleName
assert objarr[2].GetString().Data()==str(sample.downscale)
elist = elistFile.Get("elist")
return ( elist, elistFile )
def createGenerator(self,end,downscale=1):
i = downscale - 1
while i<end:
yield i
i += downscale
def fillall(self,sample):
for itree in range(len(sample.names)):
tree = sample.getchain(itree)
# print sample.name,itree
# print tree.GetEntries()
nentries = tree.GetEntries()
nprog = nentries
downscale = sample.downscale
iterator = self.createGenerator(tree.GetEntries(),sample.downscale)
if self.readElist or self.writeElist:
elist, elistFile = self.prepareElist(sample,sample.names[itree])
if self.readElist:
iterator = self.createGenerator(elist.GetN())
nprog = elist.GetN()
self.timers[6].start()
eh = EventHelper(tree)
self.timers[6].stop()
# for iev in range(tree.GetEntries()):
nall = 0
nsel = 0
# self.timers[7].start(paused=True)
# self.timers[8].start(paused=True)
if self.progress:
self.progress.initialize(length=nprog,title=sample.name+" "+sample.names[itree])
for iev in iterator:
# for iev in sample.getentries(tree):
# if sample.downscale==1 or (iev%sample.downscale)==0:
jev = iev if not self.readElist else elist.GetEntry(iev)
# self.timers[8].resume()
self.timers[8].start()
eh.getEntry(jev)
# self.timers[8].pause()
self.timers[8].stop()
nall += 1
if self.progress and nall%100==0:
self.progress.increment(100)
if self.readElist or ( \
( self.preselection==None or self.preselection.accept(eh,sample) ) and \
( sample.filter==None or sample.filter.accept(eh) ) ):
# if sample.name.startswith("TTJets"):
# print "Accepted for ",sample.name,":",eh.get("run"),eh.get("lumi"),eh.get("evt")
# self.timers[7].resume()
self.timers[7].start()
self.fill(eh,sample,itree)
# self.timers[7].pause()
self.timers[7].stop()
for timer in self.timers[:7]:
if timer.active:
timer.stop()
if self.writeElist:
elist.Enter(iev)
nsel += 1
# print "Ntot for ",sample.name,sample.names[itree]," = ",nall,nsel
# for ev in tree:
# self.fill(ev)
# self.timers[7].stop()
# self.timers[8].stop()
if self.writeElist:
elist.Write()
if self.writeElist or self.readElist:
elistFile.Close()
if self.progress and self.progress.status!=3:
self.progress.update(1.)
# handle under- & overflows
for n,v in PlotsBase.variables.iteritems():
v.moveUnderOverFlow(self.histogramList[n])
self.showTimers()
def fill1DBySign(self,name,pdg,value,weight):
fullname = name
if pdg>0:
fullname += "Minus"
elif pdg<0:
fullname += "Plus"
self.histogramList[fullname].Fill(value,weight)
def fill1DByFlavour(self,name,pdg,value,weight):
self.histogramList[name].Fill(value,weight)
if pdg!=None:
if pdg==0 or abs(pdg)==11:
self.histogramList[name+"Ele"].Fill(value,weight)
if pdg==0 or abs(pdg)==13:
self.histogramList[name+"Mu"].Fill(value,weight)
def fill1DByCategory(self,name,category,value,weight):
self.histogramList[name].Fill(value,weight)
if category!=None:
self.histogramList[name+category].Fill(value,weight)
def fill1D(self,name,value,weight):
self.fill1DBySign(name,0,value,weight)
def fill2DBySign(self,name,pdg,xvalue,yvalue,weight):
fullname = name + "_"
if pdg>0:
fullname += "Minus"
elif pdg<0:
fullname += "Plus"
self.histogramList[fullname].Fill(xvalue,yvalue,weight)
def fill2D(self,name,xvalue,yvalue,weight):
self.fill2DBySign(name,0,xvalue,yvalue,weight)
def passedCut(self,label,w,nameFlow="DefaultCutFlow"):
flow = PlotsBase.cutflows[nameFlow]
self.cutflowList[nameFlow].Fill(flow.index(label),w)
def passedCutByFlavour(self,label,pdg,w,nameFlow="DefaultCutFlow"):
flow = PlotsBase.cutflows[nameFlow]
self.cutflowList[nameFlow].Fill(flow.index(label),w)
if pdg!=None:
if pdg==0 or abs(pdg)==11:
flow = PlotsBase.cutflows[nameFlow+"Ele"]
self.cutflowList[nameFlow+"Ele"].Fill(flow.index(label),w)
if pdg==0 or abs(pdg)==13:
flow = PlotsBase.cutflows[nameFlow+"Mu"]
self.cutflowList[nameFlow+"Mu"].Fill(flow.index(label),w)
def passedCutByCategory(self,label,category,w,nameFlow="DefaultCutFlow"):
flow = PlotsBase.cutflows[nameFlow]
self.cutflowList[nameFlow].Fill(flow.index(label),w)
if category!=None and category!="":
flow = PlotsBase.cutflows[nameFlow+category]
self.cutflowList[nameFlow+category].Fill(flow.index(label),w)
def optimap(f_xy, X_params, Y_params, heat=True, symm=False):
Stats = ["Trades", "Mean", "R | Sharpe", "R | Sterling"
] ## Around which statistical metrics are we going to optimize.
arrays = (list, tuple, dict, range, numpy.ndarray, pandas.Series
) ## Indexable arrays for "X" & "Y" parameter arguments.
assert isinstance(X_params, arrays) and isinstance(
Y_params, arrays), "{TYPE} 'X' & 'Y' params must be indexable arrays."
X_labels, Y_labels = X_params, Y_params ## "Grid" df's index/column labels may tentatively be the parameters themselves.
## However, if "X"/"Y" params' array is a dict, keys will be column labels in Grid. Values will be arguments for "f_xy".
if isinstance(X_params, dict):
X_params, X_labels = list(X_params.values()), list(X_params.keys())
if isinstance(Y_params, dict):
Y_params, Y_labels = list(Y_params.values()), list(Y_params.keys())
Grid = pandas.DataFrame(
index=X_labels,
columns=pandas.MultiIndex.from_product(iterables=(Stats, Y_labels)))
Combs = list(itertools.product(range(len(X_params)), range(
len(Y_params)))) ## Indexes of all possible "X" & "Y" pairs.
## In cases where "X" & "Y" share exact same indicator nature (e.g.: both SMA), omit repeated/swapped cases. Example:
if symm:
Combs = [(nx, ny) for nx, ny in Combs if (X_params[nx] <= Y_params[ny])
] ## "(p1, p2) = (p2, p1)". Keep just one.
Prog = ProgBar.ProgBar(steps=len(
list(Combs))) ## Create a progress bar made with characters.
## For given parameters "x" and "y", we will run the exercise, find its stats and create a 2D grid.
for nx, ny in Combs: ## For every combination of parameters.
x_param, y_param, x_label, y_label = X_params[nx], Y_params[
ny], X_labels[nx], Y_labels[ny]
try: ## Run the exercise with a function as specified.
S = f_xy(x_param, y_param).Stats[symbol][
"Return"] ## From the backtest results, keep only the returns' ".Stats".
for stat in Stats:
Grid.loc[x_label, (stat, y_label)] = S[stat]
except:
1 ## When it's impossible to calculate stats (e.g.: no signals/trades), forget about errors.
Prog.up() ## Increase progress bar.
Figure, Axes = matplotlib.pyplot.subplots(ncols=len(Stats))
## Heatmaps will display the most optimal spots in red.
Grid.replace(to_replace=[-numpy.inf, numpy.inf],
value=numpy.nan,
inplace=True)
for n, stat in enumerate(Stats): ## Infs ⇧ when denominator is 0.
lim = max(abs(Grid[stat].min().min()), abs(
Grid[stat].max().max())) * 1.25 ## Y-axes' max span for lines.
if heat:
Axes[n].contourf(*numpy.meshgrid(X_params, Y_params),
Grid[stat].values.T)
# Heatmap, 2D.
else:
Grid[stat].plot(
ylim=[-lim * (Grid[stat] < 0).any().any(),
lim], ## When no negative numbers found...
ax=Axes[n],
legend=False,
linewidth=2.5)
## ...lowest y-axis point can be 0.
Axes[n].set_title(stat, fontweight="bold")
if not (heat):
Axes[0].legend(fontsize=13) ## Add legend just to the first line plot.
matplotlib.pyplot.pause(
1e-13) ## This line avoids a (quite loooong) Tkinter warning print.
return Figure, Grid
class PlotsBase:
variables = {}
cutflows = {}
def getVariables(self):
return PlotsBase.variables
def getVariables1D(self):
return [v for v in PlotsBase.variables.values() if not v.is2D()]
def getVariables2D(self):
return [v for v in PlotsBase.variables.values() if v.is2D()]
def addVariable(self,
name,
nbins,
xmin,
xmax,
scut='l',
uselog=True,
binEdges=None):
assert name.isalnum()
assert not name in self.histogramList
if not name in PlotsBase.variables:
nb = nbins if binEdges else nbins / self.rebin
PlotsBase.variables[name] = Variable(name, nb, xmin, xmax, scut,
uselog)
h1d = PlotsBase.variables[name].createHistogram(binEdges)
self.histogramList[name] = h1d
setattr(self, "h" + name, h1d)
def getCutFlows(self):
return PlotsBase.cutflows
def addCutFlow(self, labels, nameFlow="DefaultCutFlow"):
assert nameFlow.isalnum()
assert not nameFlow in self.cutflowList
if not nameFlow in PlotsBase.cutflows:
PlotsBase.cutflows[nameFlow] = CutFlow(nameFlow, labels)
h = PlotsBase.cutflows[nameFlow].createHistogram()
self.cutflowList[nameFlow] = h
setattr(self, "c" + nameFlow, h)
def addVariablePair(self,
xname,
nbinsx,
xmin,
xmax,
yname,
nbinsy,
ymin,
ymax,
uselog=True,
suffix=None):
varPair = VariablePair(xname,nbinsx/self.rebin,xmin,xmax,yname,nbinsy/self.rebin,ymin,ymax, \
uselog,suffix)
assert not varPair.name in self.histogramList
if not varPair.name in PlotsBase.variables:
PlotsBase.variables[varPair.name] = varPair
h2d = varPair.createHistogram()
self.histogramList[varPair.name] = h2d
setattr(self, "h" + varPair.name, h2d)
def __init__(self,
name,
preselection=None,
elist=None,
elistBase="./elists",
rebin=1):
self.name = name
self.preselection = preselection
self.elist = elist
if elist != None:
self.elist = elist.lower()[0]
self.elistBase = elistBase
assert os.path.isdir(elistBase)
self.timers = []
for i in range(10):
self.timers.append(MyTimer())
self.writeElist = False
self.readElist = False
if self.preselection != None and self.elist != None:
self.preselName = self.preselection.__class__.__name__
if self.elist == "w" or self.elist == "a":
self.writeElist = True
elif self.elist == "r" or self.elist == "a":
self.readElist = True
if self.writeElist or self.readElist:
self.preselDirName = os.path.join(self.elistBase, self.preselName)
if not os.path.isdir(self.preselDirName):
os.mkdir(self.preselDirName, 0744)
self.rebin = rebin
self.progress = None
def showProgressBar(self, flag=True):
if flag and self.progress == None:
self.progress = ProgBar()
elif self.progress != None and not flag:
if self.progress.isActive():
self.progress.halt()
self.progress = None
def showTimers(self):
line = ""
for t in self.timers:
line += "{0:14.2f}".format(1000000 * t.meanTime())
# line += " " + str(t.meanTime())
print line
def prepareElist(self, sample, subSampleName):
elist = None
elistFile = None
if self.readElist or self.writeElist:
dirName = os.path.join(self.preselDirName, sample.name)
if not os.path.isdir(dirName):
os.mkdir(dirName, 0744)
elistFileName = os.path.join(dirName,
subSampleName + "_elist.root")
if self.elist == "a":
self.writeElist = False
self.readElist = False
if not os.path.exists(elistFileName):
self.writeElist = True
else:
elistTime = os.path.getmtime(elistFileName)
if elistTime <= os.path.getmtime(
self.preselection.sourcefile):
self.writeElist = True
if elistTime <= os.path.getmtime(
sample.fullname(subSampleName)):
self.writeElist = True
self.readElist = not self.writeElist
if self.writeElist:
print "(Re)creating elist for ", sample.name, subSampleName
if self.writeElist:
print "Reading elist for ", sample.name, subSampleName
opt = "recreate" if self.writeElist else "read"
elistFile = ROOT.TFile(elistFileName, opt)
objarr = ROOT.TObjArray()
if self.writeElist:
objstr = ROOT.TObjString()
objstr.SetString(sample.name)
objarr.Add(objstr.Clone())
objstr.SetString(subSampleName)
objarr.Add(objstr.Clone())
objstr.SetString(str(sample.downscale))
objarr.Add(objstr.Clone())
objarr.Write("file", ROOT.TObject.kSingleKey)
elist = ROOT.TEventList(
"elist", self.preselName + " / " + sample.name + " / " +
subSampleName)
else:
objarr = elistFile.Get("file")
assert objarr[0].GetString().Data() == sample.name
assert objarr[1].GetString().Data() == subSampleName
assert objarr[2].GetString().Data() == str(sample.downscale)
elist = elistFile.Get("elist")
return (elist, elistFile)
def createGenerator(self, end, downscale=1):
i = downscale - 1
while i < end:
yield i
i += downscale
def fillall(self, sample):
for itree in range(len(sample.names)):
tree = sample.getchain(itree)
# print sample.name,itree
# print tree.GetEntries()
nentries = tree.GetEntries()
nprog = nentries
downscale = sample.downscale
iterator = self.createGenerator(tree.GetEntries(),
sample.downscale)
if self.readElist or self.writeElist:
elist, elistFile = self.prepareElist(sample,
sample.names[itree])
if self.readElist:
iterator = self.createGenerator(elist.GetN())
nprog = elist.GetN()
self.timers[6].start()
eh = EventHelper(tree)
self.timers[6].stop()
# for iev in range(tree.GetEntries()):
nall = 0
nsel = 0
# self.timers[7].start(paused=True)
# self.timers[8].start(paused=True)
if self.progress:
self.progress.initialize(length=nprog,
title=sample.name + " " +
sample.names[itree])
for iev in iterator:
# for iev in sample.getentries(tree):
# if sample.downscale==1 or (iev%sample.downscale)==0:
jev = iev if not self.readElist else elist.GetEntry(iev)
# self.timers[8].resume()
self.timers[8].start()
eh.getEntry(jev)
# self.timers[8].pause()
self.timers[8].stop()
nall += 1
if self.progress and nall % 100 == 0:
self.progress.increment(100)
if self.readElist or ( \
( self.preselection==None or self.preselection.accept(eh,sample) ) and \
( sample.filter==None or sample.filter.accept(eh) ) ):
# if sample.name.startswith("TTJets"):
# print "Accepted for ",sample.name,":",eh.get("run"),eh.get("lumi"),eh.get("evt")
# self.timers[7].resume()
self.timers[7].start()
self.fill(eh, sample, itree)
# self.timers[7].pause()
self.timers[7].stop()
for timer in self.timers[:7]:
if timer.active:
timer.stop()
if self.writeElist:
elist.Enter(iev)
nsel += 1
# print "Ntot for ",sample.name,sample.names[itree]," = ",nall,nsel
# for ev in tree:
# self.fill(ev)
# self.timers[7].stop()
# self.timers[8].stop()
if self.writeElist:
elist.Write()
if self.writeElist or self.readElist:
elistFile.Close()
if self.progress and self.progress.status != 3:
self.progress.update(1.)
# handle under- & overflows
for n, v in PlotsBase.variables.iteritems():
v.moveUnderOverFlow(self.histogramList[n])
self.showTimers()
def fill1DBySign(self, name, pdg, value, weight):
fullname = name
if pdg > 0:
fullname += "Minus"
elif pdg < 0:
fullname += "Plus"
self.histogramList[fullname].Fill(value, weight)
def fill1DByFlavour(self, name, pdg, value, weight):
self.histogramList[name].Fill(value, weight)
if pdg != None:
if pdg == 0 or abs(pdg) == 11:
self.histogramList[name + "Ele"].Fill(value, weight)
if pdg == 0 or abs(pdg) == 13:
self.histogramList[name + "Mu"].Fill(value, weight)
def fill1DByCategory(self, name, category, value, weight):
self.histogramList[name].Fill(value, weight)
if category != None:
self.histogramList[name + category].Fill(value, weight)
def fill1D(self, name, value, weight):
self.fill1DBySign(name, 0, value, weight)
def fill2DBySign(self, name, pdg, xvalue, yvalue, weight):
fullname = name + "_"
if pdg > 0:
fullname += "Minus"
elif pdg < 0:
fullname += "Plus"
self.histogramList[fullname].Fill(xvalue, yvalue, weight)
def fill2D(self, name, xvalue, yvalue, weight):
self.fill2DBySign(name, 0, xvalue, yvalue, weight)
def passedCut(self, label, w, nameFlow="DefaultCutFlow"):
flow = PlotsBase.cutflows[nameFlow]
self.cutflowList[nameFlow].Fill(flow.index(label), w)
def passedCutByFlavour(self, label, pdg, w, nameFlow="DefaultCutFlow"):
flow = PlotsBase.cutflows[nameFlow]
self.cutflowList[nameFlow].Fill(flow.index(label), w)
if pdg != None:
if pdg == 0 or abs(pdg) == 11:
flow = PlotsBase.cutflows[nameFlow + "Ele"]
self.cutflowList[nameFlow + "Ele"].Fill(flow.index(label), w)
if pdg == 0 or abs(pdg) == 13:
flow = PlotsBase.cutflows[nameFlow + "Mu"]
self.cutflowList[nameFlow + "Mu"].Fill(flow.index(label), w)
def passedCutByCategory(self,
label,
category,
w,
nameFlow="DefaultCutFlow"):
flow = PlotsBase.cutflows[nameFlow]
self.cutflowList[nameFlow].Fill(flow.index(label), w)
if category != None and category != "":
flow = PlotsBase.cutflows[nameFlow + category]
self.cutflowList[nameFlow + category].Fill(flow.index(label), w)
def BasicGraphs(GraphPath, Data):
"""Creates basic graphs
Creates graphs of the energy production
Only creates the graphs if they don't already exist
Creates:
Production graph of entire period
Monthly and Daily graphs of production
Parameters
----------
GraphPath : str
Path to save the graphs.
Data : array-like
The data to graph.
"""
pcount = 0
toCreate = 0
if not os.path.isfile(GraphPath + 'Monthly/Share2018-3.png'):
MonthGraphs = 60
else:
MonthGraphs = 0
if not os.path.isdir(GraphPath + 'Daily/'):
os.makedirs(GraphPath + 'Daily')
if not os.listdir(GraphPath + 'Daily'):
DayGraphs = 150
else:
DayGraphs = 0
if not os.path.isfile(GraphPath + 'Diesel.png'):
toCreate += 1
if not os.path.isfile(GraphPath + 'Wind.png'):
toCreate += 1
if not os.path.isfile(GraphPath + 'Water.png'):
toCreate += 1
if not os.path.isfile(GraphPath + 'Total.png'):
toCreate += 1
toCreate += MonthGraphs
toCreate += DayGraphs
#toCreate += 1
progbar = ProgBar.Progbar(target=toCreate,
newline_on_end=False,
text_description='Creating overview graphs: ')
years = mdates.YearLocator() # every year
months = mdates.MonthLocator() # every month
yearsFmt = mdates.DateFormatter('%Y')
plt.rcParams["figure.figsize"] = [16, 9]
if not os.path.isfile(GraphPath + 'Diesel.png'):
pcount += 1
#print('Creating diesel overview graph')
fig, ax = plt.subplots()
ax.plot(Data.index, Data.Diesel, color=Color1)
# format the ticks
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.xaxis.set_minor_locator(months)
# round to nearest years...
datemin = np.datetime64(Data.index[0], 'Y')
datemax = np.datetime64(Data.index[-1], 'Y') + np.timedelta64(1, 'Y')
ax.set_xlim(datemin, datemax)
plt.title('Timeseries')
plt.xlabel('Time [date]')
plt.ylabel('Diesel production [% of total]')
ax.grid(which='major', alpha=1, color='black')
ax.grid(which='minor', alpha=0.05, color='black')
fig.autofmt_xdate()
plt.savefig(GraphPath + 'Diesel.png')
plt.close(fig)
progbar.update(pcount)
if not os.path.isfile(GraphPath + 'Wind.png'):
pcount += 1
#print('Creating Wind overview graph')
fig, ax = plt.subplots()
ax.plot(Data.index, Data.Wind, color=Color1)
# format the ticks
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.xaxis.set_minor_locator(months)
# round to nearest years...
datemin = np.datetime64(Data.index[0], 'Y')
datemax = np.datetime64(Data.index[-1], 'Y')\
+ np.timedelta64(1, 'Y')
ax.set_xlim(datemin, datemax)
plt.title('Timeseries')
plt.xlabel('Time [date]')
plt.ylabel('Wind production [% of total]')
ax.grid(which='major', alpha=1, color='black')
ax.grid(which='minor', alpha=0.05, color='black')
fig.autofmt_xdate()
plt.savefig(GraphPath + 'Wind.png')
plt.close(fig)
progbar.update(pcount)
if not os.path.isfile(GraphPath + 'Water.png'):
pcount += 1
#print('Creating Water overview graph')
fig, ax = plt.subplots()
ax.plot(Data.index, Data.Water, color=Color1)
# format the ticks
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.xaxis.set_minor_locator(months)
# round to nearest years...
datemin = np.datetime64(Data.index[0], 'Y')
datemax = np.datetime64(Data.index[-1], 'Y')\
+ np.timedelta64(1, 'Y')
ax.set_xlim(datemin, datemax)
plt.title('Timeseries')
plt.xlabel('Time [date]')
plt.ylabel('Water production [% of total]')
ax.grid(which='major', alpha=1, color='black')
ax.grid(which='minor', alpha=0.05, color='black')
fig.autofmt_xdate()
plt.savefig(GraphPath + 'Water.png')
plt.close(fig)
progbar.update(pcount)
if not os.path.isfile(GraphPath + 'Total.png'):
pcount += 1
#print('Creating Total overview graph')
fig, ax = plt.subplots()
ax.plot(Data.index, Data.Total, color=Color1)
# format the ticks
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.xaxis.set_minor_locator(months)
# round to nearest years...
datemin = np.datetime64(Data.index[0], 'Y')
datemax = np.datetime64(Data.index[-1], 'Y')\
+ np.timedelta64(1, 'Y')
ax.set_xlim(datemin, datemax)
plt.title('Timeseries')
plt.xlabel('Time [date]')
plt.ylabel('Total production [%]')
ax.grid(which='major', alpha=1, color='black')
ax.grid(which='minor', alpha=0.05, color='black')
fig.autofmt_xdate()
plt.savefig(GraphPath + 'Total.png')
plt.close(fig)
progbar.update(pcount)
majorTick = mdates.DayLocator(interval=7) # every month
majorFmt = mdates.DateFormatter('%d/%m')
minorTick = mdates.DayLocator() # every day
if not os.path.isdir(GraphPath + 'Monthly'):
#print('Monthly folder not found, creating it')
os.makedirs(GraphPath + 'Monthly')
if not os.path.isfile(GraphPath + 'Monthly/Share2018-3.png'):
progbar = ProgBar.Progbar(target=toCreate,
newline_on_end=False,
text_description='Creating monthly graphs: ')
month = 3
year = 3
done = False
count = 0
#print('Creating Monthly share graph [%]')
while not done:
pcount += 1
#if count % 6 == 0:
# print('\u220E', end='', flush=True)
StartPoint = Data.index.get_loc\
('201'+str(year)+'-'+str(month)+'-'+'01 00:00',method='nearest')
EndPoint = Data['201' + str(year) + '-' + str(month)].shape[0]
fig, ax = plt.subplots()
ax.plot(Data.index[StartPoint:StartPoint + EndPoint],
Data.Diesel['201' + str(year) + '-' + str(month)])
# format the ticks
ax.xaxis.set_major_locator(majorTick)
ax.xaxis.set_major_formatter(majorFmt)
ax.xaxis.set_minor_locator(minorTick)
datemin = np.datetime64(Data.index[StartPoint],
'm') # round to nearest years...
datemax = np.datetime64(Data.index[StartPoint + EndPoint - 1], 'm')
ax.set_xlim(datemin, datemax)
ax.grid(which='major', alpha=1, color='black')
ax.grid(which='minor', alpha=0.05, color='black')
fig.autofmt_xdate()
Diesel = Data.Diesel['201' + str(year) + '-' + str(month)]
Water = Data.Water['201' + str(year) + '-' + str(month)] + Diesel
Wind = Data.Wind['201' + str(year) + '-' + str(month)] + Water
plt.plot(Water, color=Color2)
plt.plot(Wind, color=Color3)
plt.fill_between(Wind.index,
Wind,
Water,
facecolor=Color3,
alpha=1)
plt.fill_between(Water.index,
Water,
Diesel,
facecolor=Color2,
alpha=1)
plt.fill_between(Diesel.index,
Diesel,
0,
facecolor=Color1,
alpha=1)
plt.title('201' + str(year) + ' - ' + monthDict[month])
plt.ylim([-1, 105])
plt.xlabel('Time [date]')
plt.ylabel('Total production [%]')
plt.legend(['Diesel', 'Water', 'Wind'])
plt.savefig(GraphPath + 'Monthly/' + 'Share201' + str(year) + '-' +
str(month) + '.png')
plt.close(fig)
progbar.update(pcount)
if month == 12:
month = 1
year = year + 1
else:
month = month + 1
if year == 8 and month == 4:
done = True
#print('')
"""
If not created, create "random" graphs of single days
This can help see the flow of a day
"""
years = mdates.YearLocator() # every year
months = mdates.MonthLocator() # every month
hours = mdates.HourLocator() # every hour
hoursM = mdates.HourLocator(interval=3) # every 3rd hour
yearsFmt = mdates.DateFormatter('%d')
hoursFmt = mdates.DateFormatter('%H:%M')
if not os.listdir(GraphPath + 'Daily'):
progbar = ProgBar.Progbar(target=toCreate,
newline_on_end=False,
text_description='Creating daily graphs: ')
#widgets = ['Creating daily share graph: ', progressbar.Percentage(),
# ' ',progressbar.Bar(marker='∎',left='[',right=']'),
# ' ', progressbar.AdaptiveETA()]
#pbar = progressbar.ProgressBar(widgets=widgets, maxval=150)
#pbar.start()
#print('')
#print('Creating Daily share graph [%]')
for count in range(0, DayGraphs):
#if count % 5 == 0:
# print('\u220E', end='', flush=True)
#pbar.update(count+1)
pcount += 1
year = np.random.randint(3, 7 + 1)
if year == 3:
month = np.random.randint(3, 12 + 1)
elif year == 8:
month = np.random.randint(1, 3 + 1)
else:
month = np.random.randint(1, 12 + 1)
if year == 3 and month == 3:
day = np.random.randint(5, 31 + 1)
elif month == 2:
day = np.random.randint(1, 28 + 1)
elif month == 4 or month == 6 or month == 9 or month == 11:
day = np.random.randint(1, 30 + 1)
else:
day = np.random.randint(1, 31 + 1)
StartPoint = Data.index.get_loc('201' + str(year) + '-' +
str(month) + '-' + str(day) +
' 00:00',
method='nearest')
EndPoint = Data['201' + str(year) + '-' + str(month) + '-' +
str(day)].shape[0]
fig, ax = plt.subplots()
ax.plot(
Data.index[StartPoint:StartPoint + EndPoint],
Data.Diesel['201' + str(year) + '-' + str(month) + '-' +
str(day)])
# format the ticks
ax.xaxis.set_major_locator(hoursM)
ax.xaxis.set_major_formatter(hoursFmt)
ax.xaxis.set_minor_locator(hours)
# round to nearest years...
datemin = np.datetime64(Data.index[StartPoint], 'm')
datemax = np.datetime64(Data.index[StartPoint + EndPoint - 1], 'm')
# + np.timedelta64(1, 'D')
ax.set_xlim(datemin, datemax)
ax.grid(which='major', alpha=1, color='black')
ax.grid(which='minor', alpha=0.05, color='black')
fig.autofmt_xdate()
Diesel = Data.Diesel['201' + str(year) + '-' + str(month) + ' - ' +
str(day)]
Water = Data.Water['201' + str(year) + '-' + str(month) + ' - ' +
str(day)] + Diesel
Wind = Data.Wind['201' + str(year) + '-' + str(month) + ' - ' +
str(day)] + Water
#plt.plot(Diesel,color=Color1)
plt.plot(Water, color=Color2)
plt.plot(Wind, color=Color3)
plt.fill_between(Wind.index,
Wind,
Water,
facecolor=Color3,
alpha=1)
plt.fill_between(Water.index,
Water,
Diesel,
facecolor=Color2,
alpha=1)
plt.fill_between(Diesel.index,
Diesel,
0,
facecolor=Color1,
alpha=1)
plt.title('201' + str(year) + ' - ' + monthDict[month] + ' - ' +
str(day))
plt.ylim([-1, 105])
plt.xlabel('Time [date]')
plt.ylabel('Total production [%]')
plt.legend(['Diesel', 'Water', 'Wind'])
plt.savefig(GraphPath + 'Daily/' + 'Share201' + str(year) + '-' +
str(month) + '-' + str(day) + '.png')
plt.close(fig)
progbar.update(pcount)
#pbar.finish()
progbar = ProgBar.Progbar(target=toCreate,
newline_on_end=False,
text_description='Done: ')
if toCreate > 0:
progbar.update(pcount)