def loadFiles(self, filenames):
"""
Loads every log file in the list of filenames and adds them to the list of logs.
"""
for i in range(len(filenames)):
f = filenames[i]
if not os.path.exists(f):
Log.error("External log not found: " + f,
Log.getCallerId("plotter"))
else:
parser = Parser(open(f, 'r'), OrderedDict(),
Log.getCallerId(str(f)))
head, tail = os.path.split(str(f))
logId = "external_" + tail
logName = "[ext] " + tail
self.putLog(logId, parser, logName=logName)
parser.parseLog()
# This is for saving the loaded logs in the project file
# (create the necessary docks if they don't exist)
if self.__settings is None:
self.__settings = {"logFiles": {logId: f}}
elif "logFiles" not in self.__settings:
self.__settings["logFiles"] = {logId: f}
else:
self.__settings["logFiles"][logId] = f
def __init__(self, netPrototxtContents):
"""Create a deployment version of the given network.
netPrototxtContents: string
The contents of the prototxt file to deploy.
"""
# create a logger id
self._logId = Log.getCallerId('deployment')
self._originalNetworkDictionary = loader.loadNet(netPrototxtContents)
# init further attributes
self._dataLayers = dict(
) # a dictionary containing all data layers. keys are the layer ids.
self._labelBlobNames = [
] # a list containing all names of blobs, that represent any labels
self._dataBlobNames = [
] # a list containing all names of blobs, that represent any input data
self._inputBlobShapes = dict(
) # the keys of this dictionary equal self._dataBlobNames.
self._deployedNetworkDictionary = copy.deepcopy(
self._originalNetworkDictionary)
self._dHelper = DictHelper(self._deployedNetworkDictionary)
# start deployment
self._createDeployedNetwork()
def _promptForAutoNetImport(self, netPath):
"""Open a message box and ask the user if he/she wants to import the net definition given in netPath.
netPath contains a net definition that has been extracted from a loaded solver definition.
"""
# check whether the specified file does exist
if os.path.isfile(netPath):
# get the file content
with open(netPath, 'r') as file:
netPrototxt = file.read()
msgBox = QMessageBox()
msgBox.setWindowTitle("Barista")
msgBox.setText(
self.tr("Solver definition contains a network reference."))
msgBox.setInformativeText(
self.tr("Do you want to import the network, too?"))
msgBox.setDetailedText("File:\n{}\n\nNet definition:\n{}".format(
netPath, netPrototxt))
msgBox.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
msgBox.setDefaultButton(QMessageBox.Yes)
ret = msgBox.exec_()
if ret == QMessageBox.Yes:
self.openNet(netPath)
else:
callerId = Log.getCallerId('file_loader')
Log.log(
"Found network reference in loaded solver definition, but the file {} does not exist."
.format(netPath), callerId)
def __init__(self, listOfLogfiles):
'''
Initializes the concatinator with a list of Logfiles
to get a list of iterators concerning to the merged logs
call concate()
'''
self.__listeners = []
events = {
Concatenator.finish:
re.compile('Optimization Done.'),
Concatenator.interrupt:
re.compile(
"((?:Snapshotting solver state to (?:binary proto|HDF5) file ).*)"
),
Concatenator.resume:
re.compile("((?<= Resuming from ).*)"),
}
for logFile in listOfLogfiles:
log_iter = self.__createIterFromFile(logFile)
parser = Parser(log_iter, events)
listener = Concatenator.SnapshotListener(logFile)
parser.addListener(listener)
try:
parser.parseLog()
self.__listeners.append(listener)
except:
callerId = Log.getCallerId("Error Parsing Log File:" +
str(logFile))
Log.error("", callerId)
def __updateLayerList(self):
""" Update the layer list with available layers found in the net
description.
"""
# getLayers
if self.__currentSessionId is not None:
session = self.__sessionDict[self.__currentSessionId]
if isinstance(session, ClientSession):
netInternal = session.loadInternalNetFile()
currentNetwork = loader.loadNet(netInternal)
layerNames = map(
lambda layer: layer["parameters"]["name"],
filter(
lambda layer: layer["type"].name() in self.ALLOWED_LAYERTYPES,
currentNetwork["layers"].values()
)
)
layerNameList = sorted(layerNames)
else:
try:
currentNetworkPath = session.getInternalNetFile()
file = open(currentNetworkPath, 'r')
currentNetwork = loader.loadNet(file.read())
# get all the names of the layers, which match the desired type
layerNames = map(
lambda layer: layer["parameters"]["name"],
filter(
lambda layer: layer["type"].name() in self.ALLOWED_LAYERTYPES,
currentNetwork["layers"].values()
)
)
layerNameList = sorted(layerNames)
except IOError:
callerId = Log.getCallerId('weight-plotter')
Log.error("Could not open the network of this session.", callerId)
layerNameList =[]
else:
layerNameList =[]
# updates the layer Combobox with the current layers
self.layerComboBox.replaceItems(layerNameList)
if self.__currentLayerName is None or self.__currentLayerName not in layerNameList:
if not layerNameList == []:
self.__currentLayerName = layerNameList[-1]
else:
self.__currentLayerName = None
self.layerComboBox.setCurrentText(self.__currentLayerName)
def errorMsg(self, errorMsg, loggerIdString=None, addStacktrace=True):
"""Show an error message in the Logger as well as in an additional GUI popup."""
# use the logger
if loggerIdString is not None:
callerId = Log.getCallerId(loggerIdString)
else:
callerId = None
Log.log(errorMsg, callerId)
# show message in the GUI
msgBox = QMessageBox()
msgBox.setWindowTitle("Barista - Error")
msgBox.setText(self.tr(errorMsg))
if addStacktrace:
stacktrace = traceback.format_exc()
msgBox.setDetailedText(stacktrace)
msgBox.setStandardButtons(QMessageBox.Ok)
_ = msgBox.exec_()
def _readFile(self):
"""Read the content of the caffe.proto file."""
import backend.barista.caffe_versions as caffeVersions
# get path to caffe.proto file
caffeProtoFilePath = caffeVersions.getAvailableVersions(
)[0].getProtopath()
if os.path.isfile(caffeProtoFilePath):
# open caffe.proto file
with open(caffeProtoFilePath) as f:
# read complete file at once
content = f.read()
else:
callerId = Log.getCallerId("parameter_descriptions")
Log.error(
"ERROR: Unable to load caffe.proto file. Parameter descriptions and deprecated flags "
"will be left empty. Did you configure the environment variable CAFFE_ROOT?",
callerId)
content = ""
return content
from backend.barista.utils.logger import Log
from gui.main_window.node_editor.items.node_item import NodeItem
# TODO:
# -try avoiding intersecting connections (arrange ordered by connectors)
callerId = Log.getCallerId('GraphLayouter')
class NodeSort():
class Node:
OFFSET = 50
CENTER_X = 0
CENTER_Y = 0
def __init__(self, guiNode, listOfConnectedNodes):
self.guiNode = guiNode
# contains only output nodes of type Node
self.listOfConnectedNodes = listOfConnectedNodes
self.height = self.guiNode.boundingRect().height()
self.width = self.guiNode.boundingRect().width()
self.x = -1 # mark pos not set yet
self.y = -1 # mark pos not set yet
self.visited = False
def setPosition(self, columnWidths, rowHeights):
xpos = -NodeSort.Node.CENTER_X
for x in range(0, self.x):
def __init__(self):
self._env = None
self._path = None
self._cursor = None
self._txn = None
self.logid = Log.getCallerId("LMDB Input")
def save(self):
try:
# get user input
netFullPath = self._textPathNet.text()
solverFullPath = self._textPathSolver.text()
solverDirPath = os.path.dirname(solverFullPath)
netInSeparateFile = (self._comboNet.currentIndex() == self.COMBO_INDEX_SEPARATE_FILE)
netPathIsValid = len(netFullPath) > 0 and os.path.exists(netFullPath) \
and os.path.isfile(netFullPath)
# ensure that the input isn't empty
if len(solverFullPath) > 0:
# ensure that the path (except the base name) does already exist
folderExists = os.path.isdir(solverDirPath)
# if it doesn't exist yet, let the user decide whether to create all missing folders
if not folderExists:
reply = QtWidgets.QMessageBox.question(self,
self.tr("Destination doesn't exist yet."),
self.tr("Do you want to create all non-existing folders "
"in the given path?"),
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if reply == QtWidgets.QMessageBox.Yes:
folderExists = True
os.makedirs(solverDirPath)
if folderExists:
# ensure that the full path does point to a file and not a folder
fileIsNoFolder = not os.path.exists(solverFullPath) or not os.path.isdir(solverFullPath)
# input is valid, go ahead and start the actual export
if fileIsNoFolder:
if netInSeparateFile:
# network path does not need to be valid, as we are not doing anything with the referenced file
# anyway: let the user decide whether an invalid path is used on purpose
if not netPathIsValid:
reply = QtWidgets.QMessageBox.question(self,
self.tr("Network path seems to be invalid."),
"Do you want to continue anyway?",
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if reply == QtWidgets.QMessageBox.Yes:
netPathIsValid = True
if netPathIsValid:
# point to selected network file
self._solver["net"] = netFullPath
# remove any other references to a network definition
if "net_param" in self._solver:
del self._solver["net_param"]
else:
# include inline definition of the network
self._solver["net_param"] = self._network
# remove any other references to a network file
if "net" in self._solver:
del self._solver["net"]
# finally, save solver prototxt
if not netInSeparateFile or netPathIsValid:
with open(solverFullPath, 'w') as file:
file.write(saver.saveSolver(self._solver))
callerId = Log.getCallerId('export_solver')
Log.log(
"The solver has been exported successfully to {}.".format(
solverFullPath
), callerId)
# add used file paths to the recent file list
if self._defaultActions is not None:
self._defaultActions.recentSolverData.addRecently(solverFullPath, solverFullPath)
if netInSeparateFile and netPathIsValid:
self._defaultActions.recentNetData.addRecently(netFullPath, netFullPath)
self.close()
else:
QtWidgets.QMessageBox.critical(self, self.tr("Can't save solver"),
self.tr("The given path points to an existing folder instead of "
"a file."))
else:
QtWidgets.QMessageBox.critical(self,
self.tr("Can't save solver"),
self.tr("Please select a valid destination for the solver file."))
except:
QtWidgets.QMessageBox.critical(self,
self.tr("Can't save solver"),
self.tr("Unknown error."))
self.close()
def getCallerId(self):
""" Return the unique caller id for this session
"""
if self.caller_id is None:
self.caller_id = Log.getCallerId('SessionController')
return self.caller_id
def getCallerId(self):
""" Return the unique caller id for this project
"""
if self.callerId is None:
self.callerId = Log.getCallerId(self.getProjectName())
return self.callerId
def loadNet(netstring):
""" Load the prototxt string "netstring" into a dictionary.
The dictionary has the following form
{
"name": "Somenetwork",
"input_dim": [1,2,1,1],
"state": {
"phase": "TRAIN"
},
...
"layers":
{
"somerandomid1": {
"type": LayerType Instance of Pooling-Layer,
"parameters": {
"pooling_param": [
"kernel_size": 23,
"engine": "DEFAULT"
]
....
"input_param": [
{"shape": {"dim": [...], .... },
{"shape": {"dim": [...], .... },
]
}
},
"somerandomid2": {"type": ..., "parameters": ....}
},
"layerOrder": ["somerandomid1", "somerandomid2", ....]
}
"""
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
# Load Protoclass for parsing
net = proto.NetParameter()
# Get DESCRIPTION for meta infos
descr = info.ParameterGroupDescriptor(net)
# "Parse" the netdefinition in prototxt-format
try:
text_format.Merge(netstring, net)
except ParseError as ex:
raise ParseException(str(ex))
params = descr.parameter().copy() # All Parameters of the network
# add logger output if deprecated layers have been found, to inform the user that those can't be parsed yet
if len(net.layers) > 0:
callerId = Log.getCallerId('protoxt-parser')
Log.log(
"The given network contains deprecated layer definitions which are not supported and will be dropped.",
callerId)
# Layers is deprecated, Layer will be handled seperatly and linked to "Layers" key
del params["layers"]
del params["layer"]
if params.has_key("layerOrder"):
raise ValueError('Key layerOrder not expected!')
# Extract every other parameters
res = _extract_param(net, params)
res["layers"], res["layerOrder"] = _load_layers(net.layer)
res = copy.deepcopy(res)
return res
def __init__(self):
self._db = []
self._path = None
self._logid = Log.getCallerId("HDF5TXT Input")
self._projectPath = None
"""
This module defines permanent solver constraints.
"""
import os
import backend
from backend.barista.utils.logger import Log
logID = Log.getCallerId('constraints/solver')
def ensureSolverConstraints(solverDictionary):
"""Ensure that all constraints for the given solverDictionary are valid.
Sets static values and removes invalid values.
"""
# The file names inside of a session are static and must not be altered by the user
if "net" not in solverDictionary or solverDictionary[
"net"] != backend.barista.session.session_utils.Paths.FILE_NAME_NET_INTERNAL:
Log.log(
"The solver property 'net' must point to the generated network file. "
"Value has been changed from '{}' to '{}'.".format(
solverDictionary["net"]
if "net" in solverDictionary else "None", backend.barista.
session.session_utils.Paths.FILE_NAME_NET_INTERNAL), logID)
solverDictionary[
"net"] = backend.barista.session.session_utils.Paths.FILE_NAME_NET_INTERNAL
# An additional net definition inside of the solver would be inconsistent to the separately handled network
if "net_param" in solverDictionary:
Log.log(
def getCallerId(self):
""" Return the unique caller id for this session
"""
if self.caller_id is None:
self.caller_id = Log.getCallerId(self._getLogFileName())
return self.caller_id
def getCallerId(self):
""" Return the unique caller id for this session
"""
if self.caller_id is None:
self.caller_id = Log.getCallerId('weight-plotter')
return self.caller_id
def __init__(self):
self._db = None
self._path = None
self.logid = Log.getCallerId("LEVELDB Input")
def __init__(self, pathOfHdf5Txt=False):
self._db = None
self._path = None
self._pathOfHdf5Txt = pathOfHdf5Txt # HDF5TXT files can contain commentary lines that are no paths
self._logid = Log.getCallerId('HDF5 Input')
def getCallerId(self):
""" Return the unique caller id for this session
"""
if self.caller_id is None:
self.caller_id = Log.getCallerId('Deployment')
return self.caller_id