def _initializeFileLogging(self):
""" Initialize logging of FloatStreamObservables into files. """
# Helper function that handles newly added FloatStreamObservables
def addRemoveObservable(observable, action):
# Check if new observable was added and if the observable is a
# FloatStreamObservable
if action == 'added' and isinstance(observable,
FloatStreamObservable):
# Register function that writes all updates into log file
logDirectory = observable.title.split(" ")[0]
logFileName = "".join(observable.title.split(" ")[1:])
logFileName += ".fso" # FloatStreamObservable
self.userDirObj.createPath([logDirectory],
refName=logDirectory,
baseRef='currentlogdir',
force=True)
logFile = LogFile(self.userDirObj,
logFileName,
baseRef=logDirectory)
def updateLogFile(time, value, *args):
logFile.addText("%s\t%s\n" % (time, value))
observable.addObserver(updateLogFile)
# For all FloatStreamObservable that have been created already,
# call fiunction addRemoveObservable
for observable in OBSERVABLES.getAllObservablesOfType(
FloatStreamObservable):
addRemoveObservable(observable, 'added')
# Observe set of FloatStreamObservable
OBSERVABLES.addObserver(addRemoveObservable)
def __init__(self, stateSpace):
super(SeventeenAndFourValuefunctionViewer, self).__init__()
self.stateSpace = stateSpace
self.states = stateSpace["count"]["dimensionValues"]
# Combo Box for selecting the observable
self.comboBox = QtGui.QComboBox(self)
self.stateActionValuesObservables = \
OBSERVABLES.getAllObservablesOfType(StateActionValuesObservable)
self.comboBox.addItems(
map(lambda x: "%s" % x.title, self.stateActionValuesObservables))
self.connect(self.comboBox, QtCore.SIGNAL('currentIndexChanged (int)'),
self._observableChanged)
# Automatically update combobox when new float stream observables
# are created during runtime
def updateComboBox(observable, action):
self.comboBox.clear()
self.stateActionValuesObservables = \
OBSERVABLES.getAllObservablesOfType(StateActionValuesObservable)
self.comboBox.addItems(
map(lambda x: "%s" % x.title,
self.stateActionValuesObservables))
OBSERVABLES.addObserver(updateComboBox)
# Create matplotlib widgets
plotWidgetValueFunction = QtGui.QWidget(self)
plotWidgetValueFunction.setMinimumSize(800, 500)
self.figValueFunction = Figure((8.0, 5.0), dpi=100)
#self.figValueFunction.subplots_adjust(left=0.01, bottom=0.04, right=0.99,
# top= 0.95, wspace=0.05, hspace=0.11)
self.axisValueFunction = self.figValueFunction.gca()
self.canvasValueFunction = FigureCanvas(self.figValueFunction)
self.canvasValueFunction.setParent(plotWidgetValueFunction)
self.hlayout = QtGui.QHBoxLayout()
self.hlayout.addWidget(plotWidgetValueFunction)
self.hlayout.addWidget(self.comboBox)
self.setLayout(self.hlayout)
# Connect to observer (has to be the last thing!!)
self.stateActionValuesObservableCallback = \
lambda valueAccessFunction, actions: self.updateValues(valueAccessFunction, actions)
if len(self.stateActionValuesObservables) > 0:
# Show per default the first observable
self.stateActionValuesObservable = self.stateActionValuesObservables[
0]
plotWidgetValueFunction.setWindowTitle(
self.stateActionValuesObservable.title)
self.stateActionValuesObservable.addObserver(
self.stateActionValuesObservableCallback)
else:
self.stateActionValuesObservable = None
def __init__(self):
super(SPBTrajectoryViewer, self).__init__()
self.lenpole = 1.0
# Get required observables
self.trajectoryObservable = \
OBSERVABLES.getAllObservablesOfType(TrajectoryObservable)[0]
# Create matplotlib widgets
plotWidget = QtGui.QWidget(self)
plotWidget.setMinimumSize(600, 500)
plotWidget.setWindowTitle("SPB Cart Viewer")
self.fig = Figure((6.0, 5.0), dpi=100)
self.axis = self.fig.gca()
self.axis.set_xlim((-3.125, 3.125))
self.axis.set_ylim((-0.5, 5.5))
self.canvas = FigureCanvas(self.fig)
self.canvas.setParent(plotWidget)
self.canvas.draw()
# Create layout
self.hlayout = QtGui.QHBoxLayout()
self.hlayout.addWidget(plotWidget)
self.setLayout(self.hlayout)
# Connect to observer (has to be the last thing!!)
self.trajectoryObservableCallback = \
lambda *transition: self._updateSamples(*transition)
self.trajectoryObservable.addObserver(
self.trajectoryObservableCallback)
def updateComboBox(observable, action):
self.comboBox.clear()
self.stateActionValuesObservables = \
OBSERVABLES.getAllObservablesOfType(StateActionValuesObservable)
self.comboBox.addItems(
map(lambda x: "%s" % x.title,
self.stateActionValuesObservables))
def _colorCriterionChanged(self, colorCriterion):
# If we changed color criterion
if colorCriterion != self.colorCriterion:
# Remove old trajectory
self._removeTrajectory()
self.rememberedSegments = []
self.legendWidget.clear()
self.colorCriterion = colorCriterion
self.valueToColorMapping = {}
if self.colorCriterion == "Q-Value":
# Register to FunctionOverStateSpaceObservable for global Q-Function
from mmlf.framework.observables import OBSERVABLES, \
FunctionOverStateSpaceObservable
for functionObservable in OBSERVABLES.getAllObservablesOfType(
FunctionOverStateSpaceObservable):
# TODO: Name of function is hard-coded
if functionObservable.title == "Option TopLevel (optimal value function)":
def updateEvalFunction(evalFunction):
self.evalFunction = evalFunction
functionObservable.addObserver(updateEvalFunction)
break
# Displaying Q-Value makes only sense when plotting at the end of
# an episode
self._drawStyleChanged("Last Episode")
# We have to remember minimal and maximal value
self.minValue = numpy.inf
self.maxValue = -numpy.inf
def updatePolicyObservableBox(viewer, action):
self.policyObservableComboBox.clear()
policyObservables = OBSERVABLES.getAllObservablesOfType(FunctionOverStateSpaceObservable)
self.policyObservableComboBox.addItems([policyObservable.title
for policyObservable in policyObservables])
if len(policyObservables) > 0:
self.selectedPolicyObservable = policyObservables[0].title
else:
self.selectedPolicyObservable = None
def updateFunctionObservableBox(viewer, action):
self.functionObservableComboBox.clear()
functionObservables = \
OBSERVABLES.getAllObservablesOfType(FunctionOverStateSpaceObservable)
self.functionObservableComboBox.addItems([
functionObservable.title
for functionObservable in functionObservables
])
if len(functionObservables) > 0:
self.selectedFunctionObservable = functionObservables[0].title
def updateFunctionObservableBox(viewer, action):
self.functionObservableComboBox.clear()
functionObservables = \
OBSERVABLES.getAllObservablesOfType(FunctionOverStateSpaceObservable)
stateActionValueObservables = \
OBSERVABLES.getAllObservablesOfType(StateActionValuesObservable)
functionObservables.extend(stateActionValueObservables)
self.functionObservableComboBox.addItems([
functionObservable.title
for functionObservable in functionObservables
])
if self.selectedFunctionObservable is None \
and len(functionObservables) > 0:
self.selectedFunctionObservable = functionObservables[0].title
else:
# Let combobox still show the selected observable
index = self.functionObservableComboBox.findText(
self.selectedFunctionObservable)
if index != -1:
self.functionObservableComboBox.setCurrentIndex(index)
def _functionObservableChanged(self, selectedFunctionObservable):
self.lock.acquire()
if self.functionObservable is not None:
# Disconnect from old function observable
self.functionObservable.removeObserver(
self.functionObservableCallback)
# Determine new observed function observable
self.selectedFunctionObservable = str(selectedFunctionObservable)
# Connect to new function observable
self.functionObservable = OBSERVABLES.getObservable(
self.selectedFunctionObservable, FunctionOverStateSpaceObservable)
if self.functionObservable is None: # Observing a StateActionValuesObservable
self.functionObservable = OBSERVABLES.getObservable(
self.selectedFunctionObservable, StateActionValuesObservable)
self.actions = None
def functionObservableCallback(evalFunction, actions):
# If we get new options to select from
if actions != self.actions:
# Update suboptionComboBox
self.actions = actions
self.suboptionComboBox.clear()
self.suboptionComboBox.addItems(
[str(action) for action in actions])
self._updateFunction(evalFunction)
self.functionObservableCallback = functionObservableCallback
self.functionObservable.addObserver(
self.functionObservableCallback)
else: # Observing a FunctionOverStateSpaceObservable
self.functionObservableCallback = \
lambda evalFunction: self._updateFunction(evalFunction)
self.functionObservable.addObserver(
self.functionObservableCallback)
self.lock.release()
def _policyObservableChanged(self, selectedPolicyObservable):
self.lock.acquire()
if self.policyObservable:
# Disconnect from old policy observable
self.policyObservable.removeObserver(self.policyObservableCallback)
# Determine new observed policy observable
self.selectedPolicyObservable = str(selectedPolicyObservable)
# Connect to new policy observable
self.policyObservable = OBSERVABLES.getObservable(self.selectedPolicyObservable,
FunctionOverStateSpaceObservable)
self.policyObservableCallback = \
lambda policyEvalFunction: self.updatePolicy(policyEvalFunction)
self.policyObservable.addObserver(self.policyObservableCallback)
self.actions = []
self.lock.release()
def _functionObservableChanged(self, selectedFunctionObservable):
self.lock.acquire()
if self.functionObservable is not None:
# Disconnect from old function observable
self.functionObservable.removeObserver(
self.functionObservableCallback)
# Determine new observed function observable
self.selectedFunctionObservable = str(selectedFunctionObservable)
# Connect to new function observable
self.functionObservable = OBSERVABLES.getObservable(
self.selectedFunctionObservable, FunctionOverStateSpaceObservable)
self.functionObservableCallback = \
lambda evalFunction: self._updateFunction(evalFunction)
self.functionObservable.addObserver(self.functionObservableCallback)
self.actions = []
self.lock.release()
def __init__(self, stateSpace):
super(TrajectoryViewer, self).__init__()
self.stateSpace = stateSpace
# Define colors for plotting
self.colors = itertools.cycle(['g', 'b', 'c', 'm', 'k', 'y'])
self.plotLines = deque()
# Combo Boxes for selecting displaced state space dimensions
self.comboBox1 = QtGui.QComboBox(self)
self.comboBox1.addItems(sorted(stateSpace.keys()))
self.comboBox2 = QtGui.QComboBox(self)
self.comboBox2.addItems(sorted(stateSpace.keys()))
self.comboBox2.setCurrentIndex(1)
self.dimension1 = sorted(self.stateSpace.keys())[0]
self.dimension2 = sorted(self.stateSpace.keys())[1]
self.connect(self.comboBox1, QtCore.SIGNAL('currentIndexChanged (int)'),
self._dimension1Changed)
self.connect(self.comboBox2, QtCore.SIGNAL('currentIndexChanged (int)'),
self._dimension2Changed)
# Slider for controlling the number of Trajectories
self.maxLines = 5
self.numberTrajectoriesSlider = QtGui.QSlider(QtCore.Qt.Horizontal, self)
self.numberTrajectoriesSlider.setValue(self.maxLines)
self.numberTrajectoriesSlider.setMinimum(1)
self.numberTrajectoriesSlider.setMaximum(20)
self.numberTrajectoriesSlider.setTickInterval(5)
self.numberTrajectoriesSlider.setTickPosition(QtGui.QSlider.TicksBelow)
self.connect(self.numberTrajectoriesSlider, QtCore.SIGNAL('sliderReleased()'),
self._changeNumTrajectories)
# Checkbox for dis-/enabling trajectory plotting
self.plottingEnabled = True
self.enabledCheckBox = QtGui.QCheckBox("Plotting Enabled")
self.enabledCheckBox.setChecked(self.plottingEnabled)
self.connect(self.enabledCheckBox, QtCore.SIGNAL('stateChanged(int)'),
self._enablingPlotting)
# Some labels
self.dimension1Label = QtGui.QLabel("Dimension X Axis")
self.dimension2Label = QtGui.QLabel("Dimension Y Axis")
self.numTrajectoriesLabel = QtGui.QLabel("Trajectories shown")
# Create matplotlib widgets
plotWidgetTrajectory = QtGui.QWidget(self)
plotWidgetTrajectory.setMinimumSize(800, 500)
self.figTrajectory = Figure((8.0, 5.0), dpi=100)
self.axisTrajectory = self.figTrajectory .gca()
self.canvasTrajectory = FigureCanvas(self.figTrajectory)
self.canvasTrajectory .setParent(plotWidgetTrajectory )
# Initialize plotting
self._reinitializePlot()
# Create layout
self.vlayout = QtGui.QVBoxLayout()
self.hlayout1 = QtGui.QHBoxLayout()
self.hlayout1.addWidget(self.dimension1Label)
self.hlayout1.addWidget(self.comboBox1)
self.hlayout2 = QtGui.QHBoxLayout()
self.hlayout2.addWidget(self.dimension2Label)
self.hlayout2.addWidget(self.comboBox2)
self.hlayout3 = QtGui.QHBoxLayout()
self.hlayout3.addWidget(self.numTrajectoriesLabel)
self.hlayout3.addWidget(self.numberTrajectoriesSlider)
self.vlayout.addLayout(self.hlayout1)
self.vlayout.addLayout(self.hlayout2)
self.vlayout.addLayout(self.hlayout3)
self.vlayout.addWidget(self.enabledCheckBox)
self.hlayout = QtGui.QHBoxLayout()
self.hlayout.addWidget(plotWidgetTrajectory)
self.hlayout.addLayout(self.vlayout)
self.setLayout(self.hlayout)
# Connect to trajectory observable
self.trajectoryObservable = \
OBSERVABLES.getAllObservablesOfType(TrajectoryObservable)[0]
self.trajectoryObservableCallback = \
lambda *transition: self.addTransition(*transition)
self.trajectoryObservable.addObserver(self.trajectoryObservableCallback)
def _initializeModelLogging(self, spec):
""" Initialize logging of function over state spaces into files. """
# Helper function that handles newly added ModelObservable
def addRemoveObservable(observable, action):
# Check if new observable was added and if the observable is a
# ModelObservable
if action == 'added' and isinstance(observable, ModelObservable):
from mmlf.gui.viewers import VIEWERS
from mmlf.gui.viewers.model_viewer import ModelViewer
if spec is not None and spec["active"]:
# Register function that writes all updates into log file
topDirectory = observable.title.split(" ")[0]
logDirectory = "".join(
observable.title.split(" ")[1:]).strip("()")
if not self.plotThisObservableFct(logDirectory):
# Plotting this observable was not specified
return
self.userDirObj.createPath([topDirectory],
refName=topDirectory,
baseRef='currentlogdir',
force=True)
self.userDirObj.createPath([logDirectory],
refName=logDirectory,
baseRef=topDirectory,
force=True)
def plotGraphicToFile(model):
if (self.episodeCounter +
1) % spec['logFrequency'] != 0:
return # Not an episode in which we plot
logFile = \
self.userDirObj.getAbsolutePath(logDirectory,
'episode_%05d.pdf' % self.episodeCounter)
if os.path.exists(logFile):
return # we do not plot several times per episode
fig = pylab.figure(0, figsize=(22, 11))
fig.clear()
# Let observable plot itself
observable.plot(
model,
fig,
stateSpace=self.environment.stateSpace,
colouring=spec['colouring'],
plotSamples=spec['plotSamples'],
minExplorationValue=spec['minExplorationValue'],
plotStateDims=spec['stateDims'],
dimValues=spec['rasterPoints'])
# Draw structure over state space
for axis in fig.axes:
if type(axis) == __import__(
"matplotlib").axes.Axes:
# Plot only into matplotlib.axes.AxesSubplot
continue
self.environment.plotStateSpaceStructure(axis)
pylab.savefig(logFile)
observable.addObserver(plotGraphicToFile)
def createModelViewer():
modelViewer = ModelViewer(observable,
self.environment.stateSpace)
observable.addObserver(lambda *_x: modelViewer.update(*_x))
return modelViewer
VIEWERS.addViewer(createModelViewer, 'ModelViewer')
# For all ModelObservable that have been created already,
# call function addRemoveObservable
for observable in OBSERVABLES.getAllObservablesOfType(ModelObservable):
addRemoveObservable(observable, 'added')
# Observe set of ModelObservable
OBSERVABLES.addObserver(addRemoveObservable)
def _initializeFunctionOverStateSpaceLogging(self, spec):
""" Initialize logging of function over state spaces into files. """
# Helper function that handles newly added FunctionOverStateSpaceObservable
def addRemoveObservable(observable, action):
# Check if new observable was added and if the observable is a
# FunctionOverStateSpaceObservable
if action == 'added' and isinstance(
observable, FunctionOverStateSpaceObservable):
# Register function that writes all updates into log file
topDirectory = observable.title.split(" ")[0]
logDirectory = "".join(
observable.title.split(" ")[1:]).strip("()")
if not self.plotThisObservableFct(logDirectory):
# Plotting this observable was not specified
return
self.userDirObj.createPath([topDirectory],
refName=topDirectory,
baseRef='currentlogdir',
force=True)
self.userDirObj.createPath([logDirectory],
refName=logDirectory,
baseRef=topDirectory,
force=True)
def plotGraphicToFile(function):
if (self.episodeCounter + 1) % spec['logFrequency'] != 0:
return # Not an episode in whcih we plot
logFile = \
self.userDirObj.getAbsolutePath(logDirectory,
'episode_%05d.pdf' % self.episodeCounter)
if os.path.exists(logFile):
return # we do not plot several times per episode
fig = pylab.figure(0, figsize=(22, 11))
fig.clear()
# Let observable plot itself
observable.plot(function,
fig,
stateSpace=self.environment.stateSpace,
actionSpace=self.environment.actionSpace,
plotStateDims=spec['stateDims'],
rasterPoints=spec['rasterPoints'])
# Draw structure over state space
for axis in fig.axes:
if type(axis) == __import__("matplotlib").axes.Axes:
# Plot only into matplotlib.axes.AxesSubplot
continue
self.environment.plotStateSpaceStructure(axis)
pylab.savefig(logFile)
observable.addObserver(plotGraphicToFile)
# For all FunctionOverStateSpaceObservable that have been created already,
# call function addRemoveObservable
for observable in OBSERVABLES.getAllObservablesOfType(
FunctionOverStateSpaceObservable):
addRemoveObservable(observable, 'added')
# Observe set of FunctionOverStateSpaceObservable
OBSERVABLES.addObserver(addRemoveObservable)
def __init__(self, pinballMazeEnv, stateSpace):
super(PinballMazeFunctionViewer, self).__init__()
self.pinballMazeEnv = pinballMazeEnv
self.stateSpace = stateSpace
self.actions = []
self.updateCounter = 0
self.updatePlotNow = False
self.evalFunction = None
self.lock = threading.Lock()
# Create matplotlib widgets
plotWidget = QtGui.QWidget(self)
plotWidget.setMinimumSize(600, 500)
plotWidget.setWindowTitle("Pinball Maze")
self.fig = Figure((6.0, 5.0), dpi=100)
self.axis = self.fig.gca()
self.pinballMazeEnv.plotStateSpaceStructure(self.axis)
self.canvas = FigureCanvas(self.fig)
self.canvas.setParent(plotWidget)
self.canvas.draw()
self.plottedPatches = []
# Add a combobox for selecting the function over state space that is observed
self.selectedFunctionObservable = None
self.functionObservableLabel = QtGui.QLabel(
"Function over State Space")
self.functionObservableComboBox = QtGui.QComboBox(self)
functionObservables = OBSERVABLES.getAllObservablesOfType(
FunctionOverStateSpaceObservable)
self.functionObservableComboBox.addItems([
functionObservable.title
for functionObservable in functionObservables
])
if len(functionObservables) > 0:
self.selectedFunctionObservable = functionObservables[0].title
self.connect(self.functionObservableComboBox,
QtCore.SIGNAL('activated (const QString&)'),
self._functionObservableChanged)
# Automatically update funtion observable combobox when new observables
# are created during runtime
def updateFunctionObservableBox(viewer, action):
self.functionObservableComboBox.clear()
functionObservables = \
OBSERVABLES.getAllObservablesOfType(FunctionOverStateSpaceObservable)
self.functionObservableComboBox.addItems([
functionObservable.title
for functionObservable in functionObservables
])
if len(functionObservables) > 0:
self.selectedFunctionObservable = functionObservables[0].title
OBSERVABLES.addObserver(updateFunctionObservableBox)
# Slider that controls the granularity of the plot-grid
self.gridNodesPerDim = 50
self.gridNodesSlider = QtGui.QSlider(QtCore.Qt.Horizontal, self)
self.gridNodesSlider.setValue(self.gridNodesPerDim)
self.gridNodesSlider.setMinimum(0)
self.gridNodesSlider.setMaximum(100)
self.gridNodesSlider.setTickInterval(10)
self.gridNodesSlider.setTickPosition(QtGui.QSlider.TicksBelow)
self.connect(self.gridNodesSlider, QtCore.SIGNAL('sliderReleased()'),
self._changeGridNodes)
self.gridNodesLabel = QtGui.QLabel("Grid Nodes Per Dimension: %s" %
self.gridNodesPerDim)
# Slider that controls the frequency of update the plot
self.updateFrequency = 0.0
self.updateFrequencySlider = QtGui.QSlider(QtCore.Qt.Horizontal, self)
self.updateFrequencySlider.setValue(int(self.updateFrequency * 100))
self.updateFrequencySlider.setMinimum(0)
self.updateFrequencySlider.setMaximum(100)
self.updateFrequencySlider.setTickInterval(0.1)
self.updateFrequencySlider.setTickPosition(QtGui.QSlider.TicksBelow)
self.connect(self.updateFrequencySlider,
QtCore.SIGNAL('sliderReleased()'),
self._changeUpdateFrequency)
self.updateFrequencyLabel = QtGui.QLabel("UpdateFrequency: %s" %
self.updateFrequency)
# Button to enforce update of plot
self.updatePlotButton = QtGui.QPushButton("Update Plot")
self.connect(self.updatePlotButton, QtCore.SIGNAL('clicked()'),
self._updatePlot)
# Chosen xvel and yvel values
self.xVel = 0.5
self.xVelSlider = QtGui.QSlider(QtCore.Qt.Horizontal, self)
self.xVelSlider.setValue(int(self.xVel * 10))
self.xVelSlider.setMinimum(0)
self.xVelSlider.setMaximum(10)
self.xVelSlider.setTickInterval(1)
self.xVelSlider.setTickPosition(QtGui.QSlider.TicksBelow)
self.connect(self.xVelSlider, QtCore.SIGNAL('sliderReleased()'),
self._changeXVel)
self.xVelLabel = QtGui.QLabel("xvel value: %s" % self.xVel)
self.yVel = 0.5
self.yVelSlider = QtGui.QSlider(QtCore.Qt.Horizontal, self)
self.yVelSlider.setValue(int(self.yVel * 10))
self.yVelSlider.setMinimum(0)
self.yVelSlider.setMaximum(10)
self.yVelSlider.setTickInterval(1)
self.yVelSlider.setTickPosition(QtGui.QSlider.TicksBelow)
self.connect(self.yVelSlider, QtCore.SIGNAL('sliderReleased()'),
self._changeYVel)
self.yVelLabel = QtGui.QLabel("yvel value: %s" % self.xVel)
# Legend of plot
self.legendLabel = QtGui.QLabel("Legend:")
self.legendWidget = QtGui.QListWidget(self)
self.hlayout = QtGui.QHBoxLayout()
self.hlayout.addWidget(plotWidget)
self.vlayout = QtGui.QVBoxLayout()
self.functionObservableLayout = QtGui.QHBoxLayout()
self.functionObservableLayout.addWidget(self.functionObservableLabel)
self.functionObservableLayout.addWidget(
self.functionObservableComboBox)
self.vlayout.addLayout(self.functionObservableLayout)
self.gridNodesLayout = QtGui.QHBoxLayout()
self.gridNodesLayout.addWidget(self.gridNodesLabel)
self.gridNodesLayout.addWidget(self.gridNodesSlider)
self.vlayout.addLayout(self.gridNodesLayout)
self.updateFrequencyLayout = QtGui.QHBoxLayout()
self.updateFrequencyLayout.addWidget(self.updateFrequencyLabel)
self.updateFrequencyLayout.addWidget(self.updateFrequencySlider)
self.vlayout.addLayout(self.updateFrequencyLayout)
self.vlayout.addWidget(self.updatePlotButton)
self.xVelLayout = QtGui.QHBoxLayout()
self.xVelLayout.addWidget(self.xVelLabel)
self.xVelLayout.addWidget(self.xVelSlider)
self.vlayout.addLayout(self.xVelLayout)
self.yVelLayout = QtGui.QHBoxLayout()
self.yVelLayout.addWidget(self.yVelLabel)
self.yVelLayout.addWidget(self.yVelSlider)
self.vlayout.addLayout(self.yVelLayout)
self.vlayout.addWidget(self.legendLabel)
self.vlayout.addWidget(self.legendWidget)
self.hlayout.addLayout(self.vlayout)
self.setLayout(self.hlayout)
# Connect to observer (has to be the last thing!!)
self.functionObservable = None
if self.selectedFunctionObservable:
self.functionObservable = \
OBSERVABLES.getObservable(self.selectedFunctionObservable,
FunctionOverStateSpaceObservable)
self.functionObservableCallback = \
lambda evalFunction: self._updateFunction(evalFunction)
self.functionObservable.addObserver(
self.functionObservableCallback)
def updateComboBox(observable, action):
self.comboBox.clear()
self.floatStreamObservables = \
OBSERVABLES.getAllObservablesOfType(FloatStreamObservable)
self.comboBox.addItems(
map(lambda x: "%s" % x.title, self.floatStreamObservables))
def __call__(self,
manager,
observableQueue,
updateQueue,
forkQueue,
worldNumber,
runNumber,
useGUI=True):
print "", # This has to be here. Whyever???
# Choose different random seed for each run
import random
import numpy.random
random.seed(runNumber)
numpy.random.seed(runNumber)
# Inform parent process via updateQueue that we actually forked
forkQueue.put((worldNumber, runNumber))
# Give global agent and run numbers as attributes to
# threads since they are later on required by experiment
# viewer and observables
ct = currentThread()
ct.worldNumber = worldNumber
ct.runNumber = runNumber
# Add an observer for OBSERVABLES that adds an ObservableProxy
# for each Observable and puts this ObservableProxy into
# the queue to the main process
from mmlf.framework.observables import OBSERVABLES, FloatStreamObservable
from mmlf.gui.multiprocessing_observables_utils import ObservableProxy
def registerToObservables(observable, action):
if isinstance(observable, FloatStreamObservable) \
and action == 'added':
observableProxy = ObservableProxy(observable, manager)
observableQueue.put(observableProxy)
time.sleep(
0.1
) # TODO: Why is this necessary?? Somehow related to queues and spawning/forking?
observableProxy.setObserverQueue(updateQueue)
OBSERVABLES.addObserver(registerToObservables)
# Load world
world = mmlf.loadWorld(worldConfigObject=self.worldConfigObject,
useGUI=useGUI,
keepObservers=[registerToObservables])
# Set log level
mmlf.setupConsoleLogging(level="warning")
# Run specified number of episodes in the world
mmlf.log.debug("Running %s episodes in world..." %
self.numberOfEpisodes)
try:
world.run(self.numberOfEpisodes)
except Exception, e:
errorMessage = e.__class__.__name__ + ": " + str(e)
if self.exceptionOccurredSignal:
self.exceptionOccurredSignal.emit(errorMessage)
raise
def __init__(self, maze, stateSpace):
super(Maze2DFunctionViewer, self).__init__()
self.maze = maze
self.stateSpace = stateSpace
self.updateCounter = 0
self.updatePlotNow = False
self.evalFunction = None
self.lock = threading.Lock()
# Create matplotlib widgets
plotWidget = QtGui.QWidget(self)
plotWidget.setMinimumSize(600, 500)
plotWidget.setWindowTitle("Maze2D")
self.fig = Figure((6.0, 5.0), dpi=100)
self.axis = self.fig.gca()
self.axis.clear()
self.maze.drawIntoAxis(self.axis)
self.canvas = FigureCanvas(self.fig)
self.canvas.setParent(plotWidget)
self.canvas.draw()
self.plottedPatches = []
# Add a combobox for selecting the function over state space that is observed
self.selectedFunctionObservable = None
self.functionObservableLabel = QtGui.QLabel(
"Function over State Space")
self.functionObservableComboBox = QtGui.QComboBox(self)
functionObservables = OBSERVABLES.getAllObservablesOfType(
FunctionOverStateSpaceObservable)
stateActionValueObservables = \
OBSERVABLES.getAllObservablesOfType(StateActionValuesObservable)
functionObservables.extend(stateActionValueObservables)
self.functionObservableComboBox.addItems([
functionObservable.title
for functionObservable in functionObservables
])
if len(functionObservables) > 0:
self.selectedFunctionObservable = functionObservables[0].title
self.connect(self.functionObservableComboBox,
QtCore.SIGNAL('activated (const QString&)'),
self._functionObservableChanged)
# Automatically update funtion observable combobox when new observables
# are created during runtime
def updateFunctionObservableBox(viewer, action):
self.functionObservableComboBox.clear()
functionObservables = \
OBSERVABLES.getAllObservablesOfType(FunctionOverStateSpaceObservable)
stateActionValueObservables = \
OBSERVABLES.getAllObservablesOfType(StateActionValuesObservable)
functionObservables.extend(stateActionValueObservables)
self.functionObservableComboBox.addItems([
functionObservable.title
for functionObservable in functionObservables
])
if self.selectedFunctionObservable is None \
and len(functionObservables) > 0:
self.selectedFunctionObservable = functionObservables[0].title
else:
# Let combobox still show the selected observable
index = self.functionObservableComboBox.findText(
self.selectedFunctionObservable)
if index != -1:
self.functionObservableComboBox.setCurrentIndex(index)
OBSERVABLES.addObserver(updateFunctionObservableBox)
# Add a combobox for for selecting the suboption that is used when
# a StateActionValuesObservable is observed
self.selectedSuboption = None
self.suboptionLabel = QtGui.QLabel("Suboption")
self.suboptionComboBox = QtGui.QComboBox(self)
# Slider that controls the frequency of update the plot
self.updateFrequency = 0.0
self.updateFrequencySlider = QtGui.QSlider(QtCore.Qt.Horizontal, self)
self.updateFrequencySlider.setValue(int(self.updateFrequency * 100))
self.updateFrequencySlider.setMinimum(0)
self.updateFrequencySlider.setMaximum(100)
self.updateFrequencySlider.setTickInterval(0.1)
self.updateFrequencySlider.setTickPosition(QtGui.QSlider.TicksBelow)
self.connect(self.updateFrequencySlider,
QtCore.SIGNAL('sliderReleased()'),
self._changeUpdateFrequency)
self.updateFrequencyLabel = QtGui.QLabel("UpdateFrequency: %s" %
self.updateFrequency)
# Button to enforce update of plot
self.updatePlotButton = QtGui.QPushButton("Update Plot")
self.connect(self.updatePlotButton, QtCore.SIGNAL('clicked()'),
self._updatePlot)
# Legend of plot
self.legendLabel = QtGui.QLabel("Legend:")
self.legendWidget = QtGui.QListWidget(self)
self.hlayout = QtGui.QHBoxLayout()
self.hlayout.addWidget(plotWidget)
self.vlayout = QtGui.QVBoxLayout()
self.functionObservableLayout = QtGui.QHBoxLayout()
self.functionObservableLayout.addWidget(self.functionObservableLabel)
self.functionObservableLayout.addWidget(
self.functionObservableComboBox)
self.vlayout.addLayout(self.functionObservableLayout)
self.suboptionLayout = QtGui.QHBoxLayout()
self.suboptionLayout.addWidget(self.suboptionLabel)
self.suboptionLayout.addWidget(self.suboptionComboBox)
self.vlayout.addLayout(self.suboptionLayout)
self.updateFrequencyLayout = QtGui.QHBoxLayout()
self.updateFrequencyLayout.addWidget(self.updateFrequencyLabel)
self.updateFrequencyLayout.addWidget(self.updateFrequencySlider)
self.vlayout.addLayout(self.updateFrequencyLayout)
self.vlayout.addWidget(self.updatePlotButton)
self.vlayout.addWidget(self.legendLabel)
self.vlayout.addWidget(self.legendWidget)
self.hlayout.addLayout(self.vlayout)
self.setLayout(self.hlayout)
# Connect to observer (has to be the last thing!!)
self.functionObservable = None
if self.selectedFunctionObservable is not None:
self._functionObservableChanged(self.selectedFunctionObservable)
def __init__(self, pinballMazeEnv, stateSpace):
super(PinballMazeTrajectoryViewer, self).__init__()
self.pinballMazeEnv = pinballMazeEnv
self.dimensions = [stateSpace[dimName] for dimName in sorted(stateSpace.keys())]
# The segments that are obtained while drawing is disabled. These
# segment are drawn one drawing is reenabled
self.rememberedSegments = []
# The eval function that can be used for coloring the trajectory
self.evalFunction = None
self.colorsCycle = cycle([(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0),
(1.0, 1.0, 0.0), (0.0, 1.0, 1.0), (1.0, 0.0, 1.0),
(0.5, 0.0, 0.0), (0.0, 0.5, 0.0), (0.0, 0.0, 0.5)])
self.colors = defaultdict(lambda : self.colorsCycle.next())
self.valueToColorMapping = dict()
# Get required observables
self.trajectoryObservable = \
OBSERVABLES.getAllObservablesOfType(TrajectoryObservable)[0]
# Create matplotlib widgets
plotWidget = QtGui.QWidget(self)
plotWidget.setMinimumSize(600, 500)
plotWidget.setWindowTitle("Pinball Maze")
self.fig = Figure((6.0, 5.0), dpi=100)
self.axis = self.fig.gca()
self.pinballMazeEnv.plotStateSpaceStructure(self.axis)
self.canvas = FigureCanvas(self.fig)
self.canvas.setParent(plotWidget)
self.canvas.draw()
self.ballPatch = None
self.linePatches = []
# Add other elements to GUI
self.drawingEnabledCheckbox = \
QtGui.QCheckBox("Drawing enabled", self)
self.drawingEnabledCheckbox.setChecked(True)
self.drawStyle = "Current Position"
self.drawStyleLabel = QtGui.QLabel("Draw style")
self.drawStyleComboBox = QtGui.QComboBox(self)
self.drawStyleComboBox.addItems(["Current Position", "Last Episode",
"Online (All)"])
self.connect(self.drawStyleComboBox,
QtCore.SIGNAL('activated (const QString&)'),
self._drawStyleChanged)
self.colorCriterion = "Action"
self.colorCriterionLabel = QtGui.QLabel("Coloring of trajectory")
self.colorCriterionComboBox = QtGui.QComboBox(self)
self.colorCriterionComboBox.addItems(["Action", "Reward", "Q-Value"])
self.connect(self.colorCriterionComboBox,
QtCore.SIGNAL('activated (const QString&)'),
self._colorCriterionChanged)
# Legend of plot
self.legendLabel = QtGui.QLabel("Legend:")
self.legendWidget = QtGui.QListWidget(self)
# Create layout
self.hlayout = QtGui.QHBoxLayout()
self.hlayout.addWidget(plotWidget)
self.vlayout = QtGui.QVBoxLayout()
self.vlayout.addWidget(self.drawingEnabledCheckbox)
self.drawStyleLayout = QtGui.QHBoxLayout()
self.drawStyleLayout.addWidget(self.drawStyleLabel)
self.drawStyleLayout.addWidget(self.drawStyleComboBox)
self.vlayout.addLayout(self.drawStyleLayout)
self.coloringLayout = QtGui.QHBoxLayout()
self.coloringLayout.addWidget(self.colorCriterionLabel)
self.coloringLayout.addWidget(self.colorCriterionComboBox)
self.vlayout.addLayout(self.coloringLayout)
self.vlayout.addWidget(self.legendLabel)
self.vlayout.addWidget(self.legendWidget)
self.hlayout.addLayout(self.vlayout)
self.setLayout(self.hlayout)
# Connect to observer (has to be the last thing!!)
self.trajectoryObservableCallback = \
lambda *transition: self._updateSamples(*transition)
self.trajectoryObservable.addObserver(self.trajectoryObservableCallback)
def __init__(self, maze, stateSpace, actions):
super(Maze2DDetailedViewer, self).__init__()
self.maze = maze
self.stateSpace = stateSpace
self.actions = actions
self.samples = defaultdict(lambda: 0)
self.valueAccessFunction = None
self.redrawRequested = False
# Get required observables
self.trajectoryObservable = \
OBSERVABLES.getAllObservablesOfType(TrajectoryObservable)[0]
self.stateActionValuesObservables = \
OBSERVABLES.getAllObservablesOfType(StateActionValuesObservable)
# Combo Box for selecting the observable
self.observableLabel = QtGui.QLabel("Observable")
self.comboBox = QtGui.QComboBox(self)
self.comboBox.addItems(
map(lambda x: "%s" % x.title, self.stateActionValuesObservables))
self.connect(self.comboBox, QtCore.SIGNAL('currentIndexChanged (int)'),
self._observableChanged)
# Automatically update combobox when new float stream observables
# are created during runtime
def updateComboBox(observable, action):
self.comboBox.clear()
self.stateActionValuesObservables = \
OBSERVABLES.getAllObservablesOfType(StateActionValuesObservable)
self.comboBox.addItems(
map(lambda x: "%s" % x.title,
self.stateActionValuesObservables))
OBSERVABLES.addObserver(updateComboBox)
# Combo Box for selecting the updateFrequency
self.updateFreqLabel = QtGui.QLabel("Update")
self.updateComboBox = QtGui.QComboBox(self)
self.updateComboBox.addItems(["Every Episode", "Every Step"])
# Create matplotlib widgets
plotWidgetPolicy = QtGui.QWidget(self)
plotWidgetPolicy.setMinimumSize(300, 400)
plotWidgetPolicy.setWindowTitle("Policy")
self.figPolicy = Figure((3.0, 4.0), dpi=100)
self.figPolicy.subplots_adjust(left=0.01,
bottom=0.01,
right=0.99,
top=0.99,
wspace=0.05,
hspace=0.11)
self.canvasPolicy = FigureCanvas(self.figPolicy)
self.canvasPolicy.setParent(plotWidgetPolicy)
ax = self.figPolicy.gca()
ax.clear()
self.maze.drawIntoAxis(ax)
self.plotWidgetValueFunction = dict()
self.figValueFunction = dict()
self.canvasValueFunction = dict()
for index, action in enumerate(self.actions):
self.plotWidgetValueFunction[action] = QtGui.QWidget(self)
self.plotWidgetValueFunction[action].setMinimumSize(300, 400)
self.plotWidgetValueFunction[action].setWindowTitle(str(action))
self.figValueFunction[action] = Figure((3.0, 4.0), dpi=100)
self.figValueFunction[action].subplots_adjust(left=0.01,
bottom=0.01,
right=0.99,
top=0.99,
wspace=0.05,
hspace=0.11)
self.canvasValueFunction[action] = FigureCanvas(
self.figValueFunction[action])
self.canvasValueFunction[action].setParent(
self.plotWidgetValueFunction[action])
ax = self.figValueFunction[action].gca()
ax.clear()
self.maze.drawIntoAxis(ax)
self.textInstances = dict()
self.arrowInstances = []
self.canvasPolicy.draw()
for index, action in enumerate(self.actions):
self.canvasValueFunction[action].draw()
self.mdiArea = QtGui.QMdiArea(self)
self.mdiArea.addSubWindow(plotWidgetPolicy)
for index, action in enumerate(self.actions):
self.mdiArea.addSubWindow(self.plotWidgetValueFunction[action])
self.vlayout = QtGui.QVBoxLayout()
self.vlayout.addWidget(self.mdiArea)
self.hlayout = QtGui.QHBoxLayout()
self.hlayout.addWidget(self.observableLabel)
self.hlayout.addWidget(self.comboBox)
self.hlayout.addWidget(self.updateFreqLabel)
self.hlayout.addWidget(self.updateComboBox)
self.vlayout.addLayout(self.hlayout)
self.setLayout(self.vlayout)
# Connect to observer (has to be the last thing!!)
self.trajectoryObservableCallback = \
lambda *transition: self.updateSamples(*transition)
self.trajectoryObservable.addObserver(
self.trajectoryObservableCallback)
self.stateActionValuesObservableCallback = \
lambda valueAccessFunction, actions: self.updateValues(valueAccessFunction, actions)
if len(self.stateActionValuesObservables) > 0:
# Show per default the first observable
self.stateActionValuesObservable = self.stateActionValuesObservables[
0]
self.stateActionValuesObservable.addObserver(
self.stateActionValuesObservableCallback)
else:
self.stateActionValuesObservable = None
def __init__(self):
super(FloatStreamViewer, self).__init__()
# Create matplotlib widget
plotWidget = QtGui.QWidget(self)
plotWidget.setMinimumSize(800, 500)
fig = Figure((8.0, 5.0), dpi=100)
self.canvas = FigureCanvas(fig)
self.canvas.setParent(plotWidget)
self.axis = fig.gca()
# Local container for displayed values
self.values = deque()
self.times = deque()
# Combo Box for selecting the observable
self.comboBox = QtGui.QComboBox(self)
self.floatStreamObservables = \
OBSERVABLES.getAllObservablesOfType(FloatStreamObservable)
self.comboBox.addItems(
map(lambda x: "%s" % x.title, self.floatStreamObservables))
self.connect(self.comboBox, QtCore.SIGNAL('currentIndexChanged (int)'),
self._observableChanged)
# Automatically update combobox when new float stream observables
# are created during runtime
def updateComboBox(observable, action):
self.comboBox.clear()
self.floatStreamObservables = \
OBSERVABLES.getAllObservablesOfType(FloatStreamObservable)
self.comboBox.addItems(
map(lambda x: "%s" % x.title, self.floatStreamObservables))
OBSERVABLES.addObserver(updateComboBox)
# The number of values from the observable that are remembered
self.windowSize = 64
# Slider for controlling the window size
self.windowSizeSlider = QtGui.QSlider(QtCore.Qt.Horizontal, self)
self.windowSizeSlider.setValue(numpy.log2(self.windowSize))
self.windowSizeSlider.setMinimum(0)
self.windowSizeSlider.setMaximum(15)
self.windowSizeSlider.setTickInterval(1)
self.windowSizeSlider.setTickPosition(QtGui.QSlider.TicksBelow)
self.connect(self.windowSizeSlider, QtCore.SIGNAL('sliderReleased()'),
self._changeWindowSize)
self.windowSizeLabel = QtGui.QLabel("WindowSize: %s" % self.windowSize)
# The length of the moving window average
self.mwaSize = 10
# Slider for controlling the moving average window
self.mwaSlider = QtGui.QSlider(QtCore.Qt.Horizontal, self)
self.mwaSlider.setValue(self.mwaSize)
self.mwaSlider.setMinimum(1)
self.mwaSlider.setMaximum(50)
self.mwaSlider.setTickInterval(10)
self.mwaSlider.setTickPosition(QtGui.QSlider.TicksBelow)
self.connect(self.mwaSlider, QtCore.SIGNAL('sliderReleased()'),
self._changeMWA)
self.mwaLabel = QtGui.QLabel("Moving Window Average : %s" %
self.mwaSize)
# Create layout
self.vlayout = QtGui.QVBoxLayout()
self.vlayout.addWidget(self.comboBox)
self.vlayout.addWidget(self.windowSizeSlider)
self.vlayout.addWidget(self.windowSizeLabel)
self.vlayout.addWidget(self.mwaSlider)
self.vlayout.addWidget(self.mwaLabel)
self.hlayout = QtGui.QHBoxLayout()
self.hlayout.addWidget(plotWidget)
self.hlayout.addLayout(self.vlayout)
self.setLayout(self.hlayout)
# Handling connecting to observable
self.observableCallback = lambda time, value, *args: self.update(
time, value)
if len(self.floatStreamObservables) > 0:
# Show per default the first observable
self.observable = self.floatStreamObservables[0]
# Connect to observer (has to be the last thing!!)
self.observable.addObserver(self.observableCallback)
else:
self.observable = None
def __init__(self, stateSpace):
super(MountainCarPolicyViewer, self).__init__()
self.stateSpace = stateSpace
self.actions = []
self.colors = ['r','g','b', 'c', 'y']
self.lock = threading.Lock()
# Add a combobox for selecting the policy observable
self.policyObservableLabel = QtGui.QLabel("Policy Observable")
self.policyObservableComboBox = QtGui.QComboBox(self)
policyObservables = \
OBSERVABLES.getAllObservablesOfType(FunctionOverStateSpaceObservable)
self.policyObservableComboBox.addItems([policyObservable.title
for policyObservable in policyObservables])
self.selectedPolicyObservable = None
if len(policyObservables) > 0:
self.selectedPolicyObservable = policyObservables[0].title
self.connect(self.policyObservableComboBox,
QtCore.SIGNAL('activated (const QString&)'),
self._policyObservableChanged)
# Automatically update policy observable combobox when new observables
# are created during runtime
def updatePolicyObservableBox(viewer, action):
self.policyObservableComboBox.clear()
policyObservables = OBSERVABLES.getAllObservablesOfType(FunctionOverStateSpaceObservable)
self.policyObservableComboBox.addItems([policyObservable.title
for policyObservable in policyObservables])
if len(policyObservables) > 0:
self.selectedPolicyObservable = policyObservables[0].title
else:
self.selectedPolicyObservable = None
OBSERVABLES.addObserver(updatePolicyObservableBox)
# Get trajectory observable which is required for informing about end of episode
self.trajectoryObservable = \
OBSERVABLES.getAllObservablesOfType(TrajectoryObservable)[0]
self.episodeTerminated = False
# Slider that controls the granularity of the plot-grid
self.gridNodesPerDim = 25
self.gridNodesSlider = QtGui.QSlider(QtCore.Qt.Horizontal, self)
self.gridNodesSlider.setValue(self.gridNodesPerDim)
self.gridNodesSlider.setMinimum(0)
self.gridNodesSlider.setMaximum(100)
self.gridNodesSlider.setTickInterval(10)
self.gridNodesSlider.setTickPosition(QtGui.QSlider.TicksBelow)
self.connect(self.gridNodesSlider, QtCore.SIGNAL('sliderReleased()'),
self._changeGridNodes)
self.gridNodesLabel = QtGui.QLabel("Grid Nodes Per Dimension: %s"
% self.gridNodesPerDim )
# Create matplotlib widgets
plotWidget = QtGui.QWidget(self)
plotWidget.setMinimumSize(600, 500)
plotWidget.setWindowTitle("Policy")
self.fig = Figure((6.0, 5.0), dpi=100)
self.axis = self.fig.gca()
self.canvas = FigureCanvas(self.fig)
self.canvas.setParent(plotWidget)
# Small text in plot legend
matplotlib.rcParams.update({'legend.fontsize': 6})
self.hlayout = QtGui.QHBoxLayout()
self.hlayout.addWidget(plotWidget)
self.vlayout = QtGui.QVBoxLayout()
self.vlayout.addWidget(self.policyObservableLabel)
self.vlayout.addWidget(self.policyObservableComboBox)
self.vlayout.addWidget(self.gridNodesLabel)
self.vlayout.addWidget(self.gridNodesSlider)
self.hlayout.addLayout(self.vlayout)
self.setLayout(self.hlayout)
# Connect to observer (has to be the last thing!!)
self.trajectoryObservableCallback = \
lambda *transition: self.updateSamples(*transition)
self.trajectoryObservable.addObserver(self.trajectoryObservableCallback)
self.policyObservable = None
if self.selectedPolicyObservable:
self.policyObservable = OBSERVABLES.getObservable(self.selectedPolicyObservable,
FunctionOverStateSpaceObservable)
self.policyObservableCallback = \
lambda policyEvalFunction: self.updatePolicy(policyEvalFunction)
self.policyObservable.addObserver(self.policyObservableCallback)
