def __init__(self, shuttlingEdges=list() ):
super(ShuttlingGraph, self).__init__(shuttlingEdges)
self.currentPosition = None
self.currentPositionName = None
self.nodeLookup = dict()
self.currentPositionObservable = Observable()
self.graphChangedObservable = Observable()
self.initGraph()
self._hasChanged = True
def __init__(self, dbConnection):
self.database_conn_str = dbConnection.connectionString
self.engine = create_engine(self.database_conn_str, echo=dbConnection.echo)
self.isOpen = False
self.beginInsertRows = Observable()
self.endInsertRows = Observable()
self.beginResetModel = Observable()
self.endResetModel = Observable()
self._loadingEvents = list()
self._currentProfile = None
self._profiles = set()
def __init__(self):
numParameters = len(self.parameterNames)
self.epsfcn = 0.0
self.parameters = [0] * numParameters
self.startParameters = [1] * numParameters
self.startParameterExpressions = None # will be initialized by FitUiTableModel if values are available
self.parameterEnabled = [True] * numParameters
self.parametersConfidence = [None] * numParameters
self.units = None
self.results = SequenceDict({'RMSres': ResultRecord(name='RMSres')})
self.useSmartStartValues = False
self.hasSmartStart = not hasattr(self.smartStartValues, 'isNative')
self.parametersUpdated = Observable()
self.parameterBounds = [[None, None] for _ in range(numParameters)]
self.parameterBoundsExpressions = None
self.useErrorBars = True
def __init__(self, dbConnection):
self.database_conn_str = dbConnection.connectionString
self.engine = create_engine(self.database_conn_str,
echo=dbConnection.echo)
self.isOpen = False
self.beginInsertRows = Observable()
self.endInsertRows = Observable()
self.beginResetModel = Observable()
self.endResetModel = Observable()
self._loadingEvents = list()
self._currentProfile = None
self._profiles = set()
def __init__(self, dbConnection):
self.database_conn_str = dbConnection.connectionString
self.engine = create_engine(self.database_conn_str, echo=dbConnection.echo)
self.studies = list()
self.measurements = list()
self.measurementDict = dict() #key - startDate, val - Measurement
self.spaces = list()
self.isOpen = False
self.beginInsertMeasurement = Observable()
self.endInsertMeasurement = Observable()
self.studiesObservable = Observable()
self.measurementsUpdated = Observable()
self.scanNamesChanged = Observable()
self._scanNames = SequenceDict()
self._scanNameFilter = None
self.fromTime = datetime(2014, 11, 1, 0, 0)
self.toTime = datetime.combine((datetime.now()+timedelta(days=1)).date(), time())
class UseTracker(object):
def __init__(self):
self.adjustingDevices = set()
self.doneAdjusting = Observable()
@property
def adjusting(self):
return len(self.adjustingDevices) > 0
def take(self, device):
self.adjustingDevices.add(device)
def release(self, device):
self.adjustingDevices.discard(device)
if not self.adjusting:
self.doneAdjusting.firebare()
self.doneAdjusting.callbacks = list()
def callWhenDoneAdjusting(self, callback):
if self.adjusting:
self.doneAdjusting.subscribe(callback)
else:
QtCore.QTimer.singleShot(0, callback)
def __init__(self, config, globalDict, parent=None):
VoltageGlobalAdjustForm.__init__(self)
VoltageGlobalAdjustBase.__init__(self, parent)
self.config = config
self.configname = 'VoltageGlobalAdjust.Settings'
self.settings = self.config.get(self.configname, Settings())
self.globalAdjustDict = SequenceDict()
self.myLabelList = list()
self.myBoxList = list()
self.historyCategory = 'VoltageGlobalAdjust'
self.adjustHistoryName = None
self.globalDict = globalDict
self.adjustCache = self.config.get(self.configname + ".cache", dict())
self.savedValue = defaultdict(lambda: None)
self.displayValueObservable = defaultdict(lambda: Observable())
def __init__(self):
numParameters = len(self.parameterNames)
self.epsfcn=0.0
self.parameters = [0] * numParameters
self.startParameters = [1] * numParameters
self.startParameterExpressions = None # will be initialized by FitUiTableModel if values are available
self.parameterEnabled = [True] * numParameters
self.parametersConfidence = [None] * numParameters
self.units = None
self.results = SequenceDict({'RMSres': ResultRecord(name='RMSres')})
self.useSmartStartValues = False
self.hasSmartStart = not hasattr(self.smartStartValues, 'isNative' )
self.parametersUpdated = Observable()
self.parameterBounds = [[None, None] for _ in range(numParameters) ]
self.parameterBoundsExpressions = None
self.useErrorBars = True
class DBPersist:
store = None
name = "DB Persist"
newPersistData = Observable()
def __init__(self):
self.initialized = False
def __setstate__(self, state):
self.__dict__.update(state)
self.initialized = False
def initDB(self):
dbConnection = getProject().dbConnection
if DBPersist.store is None:
DBPersist.store = ValueHistoryStore(dbConnection)
DBPersist.store.open_session()
self.initialized = True
def persist(self, space, source, time, value, minval=None, maxval=None, unit=None):
if not self.initialized:
self.initDB()
if source:
ts = datetime.fromtimestamp(time)
DBPersist.store.add( space, source, value, unit, ts, bottom=minval, top=maxval )
self.newPersistData.fire( space=space, parameter=source, value=value, unit=unit, timestamp=ts, bottom=minval, top=maxval )
def rename(self, space, oldsourcename, newsourcename):
if not self.initialized:
self.initDB()
DBPersist.store.rename(space, oldsourcename, newsourcename)
def paramDef(self):
return []
def sourceDict(self):
if not self.initialized:
self.initDB()
return DBPersist.store.sourceDict
def __init__(self, config, globalDict, parent=None):
Form.__init__(self)
Base.__init__(self, parent)
self.config = config
self.configname = 'VoltageLocalAdjust.Settings'
self.settings = self.config.get(self.configname, Settings())
self.localAdjustList = self.config.get(self.configname + ".local",
list())
for record in self.localAdjustList:
record.globalDict = globalDict
try:
record.gain.valueChanged.connect(
partial(self.onValueChanged, record),
QtCore.Qt.UniqueConnection)
except:
pass
self.channelDict = dict(
((record.name, record) for record in self.localAdjustList))
self.historyCategory = 'VoltageLocalAdjust'
self.adjustHistoryName = None
self.globalDict = globalDict
self.savedValue = defaultdict(lambda: None)
self.displayValueObservable = defaultdict(lambda: Observable())
class LoadingHistory(object):
def __init__(self, dbConnection):
self.database_conn_str = dbConnection.connectionString
self.engine = create_engine(self.database_conn_str, echo=dbConnection.echo)
self.isOpen = False
self.beginInsertRows = Observable()
self.endInsertRows = Observable()
self.beginResetModel = Observable()
self.endResetModel = Observable()
self._loadingEvents = list()
self._currentProfile = None
self._profiles = set()
def open(self):
try:
Base.metadata.create_all(self.engine)
self.Session = sessionmaker(bind=self.engine, expire_on_commit=False)
self.session = self.Session()
self.isOpen = True
except OperationalError as e:
logging.getLogger(__name__).warning("Cannot open database: {0}".format(str(e)))
self.isOpen = False
def close(self):
self.session.commit()
self.isOpen = False
def __enter__(self):
if not self.isOpen:
self.open()
return self
def __exit__(self, exittype, value, tb):
self.session.commit()
def addLoadingEvent(self, loadingEvent):
try:
self.session.add( loadingEvent )
self.session.commit()
if self._currentProfile is None or loadingEvent.loadingProfile==self._currentProfile:
self.beginInsertRows.fire(first=len(self._loadingEvents), last=len(self._loadingEvents))
self._loadingEvents.append( loadingEvent )
self.endInsertRows.firebare()
except (InvalidRequestError, IntegrityError, ProgrammingError) as e:
logging.getLogger(__name__).warning( str(e) )
self.session.rollback()
self.session = self.Session()
def setTrappingDuration(self, duration):
self._loadingEvents[-1].trappingDuration = duration
self.commit()
def commit(self):
try:
self.session.commit()
except (InvalidRequestError, IntegrityError, ProgrammingError) as e:
logging.getLogger(__name__).warning( str(e) )
self.session.rollback()
self.session = self.Session()
def query(self, fromTime, toTime, loadingProfile):
self.beginResetModel.firebare()
self._loadingEvents[:] = [] # clear the list in place
self._loadingEvents.extend( self.session.query(LoadingEvent).filter(LoadingEvent.trappingTime>=fromTime).filter(LoadingEvent.trappingTime<=toTime).filter(LoadingEvent.loadingProfile == loadingProfile).order_by(LoadingEvent.trappingTime).all())
self.endResetModel.firebare()
self._currentProfile = loadingProfile
def getProfiles(self):
return None
@property
def loadingEvents(self):
return self._loadingEvents
def lastEvent(self):
return self._loadingEvents[-1]
class ShuttlingGraph(list):
def __init__(self, shuttlingEdges=list() ):
super(ShuttlingGraph, self).__init__(shuttlingEdges)
self.currentPosition = None
self.currentPositionName = None
self.nodeLookup = dict()
self.currentPositionObservable = Observable()
self.graphChangedObservable = Observable()
self.initGraph()
self._hasChanged = True
def initGraph(self):
self.shuttlingGraph = MultiGraph()
for edge in self:
self.shuttlingGraph.add_node(edge.startName)
self.shuttlingGraph.add_node(edge.stopName)
self.shuttlingGraph.add_edge(edge.startName, edge.stopName, key=hash(edge), edge=edge,
weight=abs(edge.stopLine-edge.startLine))
self.nodeLookup[edge.startLine] = edge.startName
self.nodeLookup[edge.stopLine] = edge.stopName
def rgenerateNodeLookup(self):
self.nodeLookup.clear()
for edge in self:
self.nodeLookup[edge.startLine] = edge.startName
self.nodeLookup[edge.stopLine] = edge.stopName
@property
def hasChanged(self):
return self._hasChanged
@hasChanged.setter
def hasChanged(self, value):
self._hasChanged = value
def position(self, line):
return self.nodeLookup.get(line)
def setPosition(self, line):
if self.currentPosition!=line:
self.currentPosition = line
self.currentPositionName = self.position(line)
self.currentPositionObservable.fire( line=line, text=firstNotNone(self.currentPositionName, "") )
def getMatchingPosition(self,graph):
"""Try to match node name/position to the current settings in the provided ShuttlingGraph."""
if not graph:
return self.currentPosition # no change
# Matching node name. Need to set the corresponding position
for edge in self:
if edge.startName == graph.currentPositionName:
return edge.startLine
if edge.stopName == graph.currentPositionName:
return edge.stopLine
#if graph.currentPosition:
# return graph.currentPosition #just use the graph's position
return self.currentPosition
def addEdge(self, edge):
self._hasChanged = True
self.append(edge)
self.shuttlingGraph.add_edge(edge.startName, edge.stopName, key=hash(edge), edge=edge, weight=abs(edge.stopLine-edge.startLine))
self.nodeLookup[edge.startLine] = edge.startName
self.nodeLookup[edge.stopLine] = edge.stopName
self.graphChangedObservable.firebare()
self.setPosition(self.currentPosition)
def isValidEdge(self, edge):
return ((edge.startLine not in self.nodeLookup or self.nodeLookup[edge.startLine] == edge.startName)
and (edge.stopLine not in self.nodeLookup or self.nodeLookup[edge.stopLine] == edge.stopName))
def getValidEdge(self):
index = 0
while self.shuttlingGraph.has_node("Start_{0}".format(index)):
index += 1
startName = "Start_{0}".format(index)
index = 0
while self.shuttlingGraph.has_node("Stop_{0}".format(index)):
index += 1
stopName = "Stop_{0}".format(index)
index = 0
startLine = (max( self.nodeLookup.keys() )+1) if self.nodeLookup else 1
stopLine = startLine + 1
return ShuttleEdge(startName, stopName, startLine, stopLine, 0, 0, 0, 0)
def removeEdge(self, edgeno):
self._hasChanged = True
edge = self.pop(edgeno)
self.shuttlingGraph.remove_edge(edge.startName, edge.stopName, hash(edge))
if self.shuttlingGraph.degree(edge.startName) == 0:
self.shuttlingGraph.remove_node(edge.startName)
if self.shuttlingGraph.degree(edge.stopName) == 0:
self.shuttlingGraph.remove_node(edge.stopName)
self.graphChangedObservable.firebare()
self.rgenerateNodeLookup()
self.setPosition(self.currentPosition)
def setStartName(self, edgeno, startName):
self._hasChanged = True
startName = str(startName)
edge = self[edgeno]
if edge.startName != startName:
self.shuttlingGraph.remove_edge(edge.startName, edge.stopName, key=hash(edge))
if self.shuttlingGraph.degree(edge.startName) == 0:
self.shuttlingGraph.remove_node(edge.startName)
edge.startName = startName
self.shuttlingGraph.add_edge(edge.startName, edge.stopName, key=hash(edge), edge=edge,
weight=abs(edge.stopLine-edge.startLine) )
self.graphChangedObservable.firebare()
self.setPosition(self.currentPosition)
self.rgenerateNodeLookup()
return True
def setStopName(self, edgeno, stopName):
self._hasChanged = True
stopName = str(stopName)
edge = self[edgeno]
if edge.stopName != stopName:
self.shuttlingGraph.remove_edge(edge.startName, edge.stopName, key=hash(edge))
if self.shuttlingGraph.degree(edge.stopName) == 0:
self.shuttlingGraph.remove_node(edge.stopName)
edge.stopName = stopName
self.shuttlingGraph.add_edge(edge.startName, edge.stopName, key=hash(edge), edge=edge,
weight=abs(edge.stopLine-edge.startLine) )
self.graphChangedObservable.firebare()
self.rgenerateNodeLookup()
self.setPosition(self.currentPosition)
return True
def setStartLine(self, edgeno, startLine):
self._hasChanged = True
edge = self[edgeno]
if startLine != edge.startLine and (startLine not in self.nodeLookup or self.nodeLookup[startLine] == edge.startName):
self.nodeLookup.pop(edge.startLine)
edge.startLine = startLine
self.shuttlingGraph.edge[edge.startName][edge.stopName][hash(edge)]['weight'] = abs(edge.stopLine-edge.startLine)
self.rgenerateNodeLookup()
self.graphChangedObservable.firebare()
self.setPosition(self.currentPosition)
return True
return False
def setStopLine(self, edgeno, stopLine):
self._hasChanged = True
edge = self[edgeno]
if stopLine != edge.stopLine and (stopLine not in self.nodeLookup or self.nodeLookup[stopLine] == edge.stopName):
self.nodeLookup.pop(edge.stopLine)
edge.stopLine = stopLine
self.shuttlingGraph.edge[edge.startName][edge.stopName][hash(edge)]['weight'] = abs(edge.stopLine-edge.startLine)
self.rgenerateNodeLookup()
self.graphChangedObservable.firebare()
self.setPosition(self.currentPosition)
return True
return False
def setIdleCount(self, edgeno, idleCount):
self._hasChanged = True
self[edgeno].idleCount = idleCount
return True
def setSteps(self, edgeno, steps):
self._hasChanged = True
self[edgeno].steps = steps
return True
def shuttlePath(self, fromName, toName ):
fromName = firstNotNone(fromName, self.currentPositionName)
fromName = fromName if fromName else self.position(float(self.currentPosition))
if fromName not in self.shuttlingGraph:
raise ShuttlingGraphException("Shuttling failed, origin '{0}' is not a valid shuttling node".format(fromName))
if toName not in self.shuttlingGraph:
raise ShuttlingGraphException("Shuttling failed, target '{0}' is not a valid shuttling node".format(toName))
sp = shortest_path(self.shuttlingGraph, fromName, toName)
path = list()
for a, b in pairs_iter(sp):
edge = sorted(self.shuttlingGraph.edge[a][b].values(), key=itemgetter('weight'))[0]['edge']
path.append((a, b, edge, self.index(edge)))
return path
def nodes(self):
return self.shuttlingGraph.nodes()
def toXmlElement(self, root):
mydict = dict( ( (key, str(getattr(self, key))) for key in ('currentPosition', 'currentPositionName') if getattr(self, key) is not None ) )
myElement = ElementTree.SubElement(root, "ShuttlingGraph", attrib=mydict )
for edge in self:
edge.toXmlElement( myElement )
return myElement
def setStartType(self, edgeno, Type):
self._hasChanged = True
self[edgeno].startType = str(Type)
return True
def setStopType(self, edgeno, Type):
self._hasChanged = True
self[edgeno].stopType = str(Type)
return True
def setStartLength(self, edgeno, length):
edge = self[edgeno]
if length!=edge.startLength:
if length+edge.stopLength<edge.sampleCount:
self._hasChanged = True
edge.startLength = int(length)
else:
return False
return True
def setStopLength(self, edgeno, length):
edge = self[edgeno]
if length!=edge.stopLength:
if edge.startLength+length<edge.sampleCount:
self._hasChanged = True
edge.stopLength = int(length)
else:
return False
return True
@staticmethod
def fromXmlElement( element ):
edgeElementList = element.findall("ShuttleEdge")
edgeList = [ ShuttleEdge.fromXmlElement(e) for e in edgeElementList ]
return ShuttlingGraph(edgeList)
def __init__(self, globalDict=None, settings= None):
Observable.__init__(self)
self.settings = settings if settings else dict()
self.globalDict = globalDict if globalDict else dict()
self.setDefault()
self.settingsName = None
class MeasurementContainer(object):
def __init__(self, dbConnection):
self.database_conn_str = dbConnection.connectionString
self.engine = create_engine(self.database_conn_str, echo=dbConnection.echo)
self.studies = list()
self.measurements = list()
self.measurementDict = dict() #key - startDate, val - Measurement
self.spaces = list()
self.isOpen = False
self.beginInsertMeasurement = Observable()
self.endInsertMeasurement = Observable()
self.studiesObservable = Observable()
self.measurementsUpdated = Observable()
self.scanNamesChanged = Observable()
self._scanNames = SequenceDict()
self._scanNameFilter = None
self.fromTime = datetime(2014, 11, 1, 0, 0)
self.toTime = datetime.combine((datetime.now()+timedelta(days=1)).date(), time())
def setScanNameFilter(self, scanNameFilter):
if self._scanNameFilter!=scanNameFilter:
self._scanNameFilter = scanNameFilter
self.query(self.fromTime, self.toTime, self._scanNameFilter)
def open(self):
Base.metadata.create_all(self.engine)
self.Session = sessionmaker(bind=self.engine, expire_on_commit=False)
self.session = self.Session()
self.isOpen = True
def close(self):
self.session.commit()
self.isOpen = False
def __enter__(self):
if not self.isOpen:
self.open()
return self
def __exit__(self, exittype, value, tb):
self.session.commit()
def addMeasurement(self, measurement):
try:
self.session.add( measurement )
self.session.commit()
self.measurementDict[str(measurement.startDate)] = measurement
if self._scanNameFilter is None or measurement.scanName in self._scanNameFilter:
self.beginInsertMeasurement.fire(first=0, last=0)
self.measurements.insert( 0, measurement )
self.endInsertMeasurement.firebare()
if measurement.scanName not in self._scanNames:
self._scanNames.setdefault( measurement.scanName, True )
self.scanNamesChanged.fire( scanNames=self._scanNames )
except (InvalidRequestError, IntegrityError, ProgrammingError) as e:
logging.getLogger(__name__).warning( str(e) )
self.session.rollback()
self.session = self.Session()
def commit(self):
try:
self.session.commit()
except (InvalidRequestError, IntegrityError, ProgrammingError) as e:
logging.getLogger(__name__).warning( str(e) )
self.session.rollback()
self.session = self.Session()
def query(self, fromTime, toTime, scanNameFilter=None):
if scanNameFilter is None:
self.measurements = self.session.query(Measurement).filter(Measurement.startDate>=fromTime).filter(Measurement.startDate<=toTime).order_by(Measurement.id.desc()).all()
self._scanNames = SequenceDict(((m.scanName, self._scanNames.get(m.scanName, True)) for m in self.measurements))
self._scanNames.sort()
else:
self.measurements = self.session.query(Measurement).filter(Measurement.startDate>=fromTime).filter(Measurement.startDate<=toTime).filter(Measurement.scanName.in_(scanNameFilter)).order_by(Measurement.id.desc()).all()
scanNames = self.session.query(Measurement.scanName).filter(Measurement.startDate>=fromTime).filter(Measurement.startDate<=toTime).group_by(Measurement.scanName).order_by(Measurement.scanName).all()
self._scanNames = SequenceDict(((name, name in scanNameFilter) for name, in scanNames))
self.scanNamesChanged.fire( scanNames=self.scanNames )
self.measurementsUpdated.fire(measurements=self.measurements)
self.fromTime, self.toTime = fromTime, toTime
def refreshLookups(self):
"""Load the basic short tables into memory
those are: Space"""
try:
self.spaces = dict(( (s.name, s) for s in self.session.query(Space).all() ))
except (InvalidRequestError, IntegrityError, ProgrammingError) as e:
logging.getLogger(__name__).warning( str(e) )
self.session.rollback()
self.session = self.Session()
def getSpace(self, name):
if name not in self.spaces:
self.refreshLookups()
if name in self.spaces:
return self.spaces[name]
s = Space(name=name)
self.spaces[name] = s
return s
@property
def scanNames(self):
return self._scanNames
def __init__(self):
self.adjustingDevices = set()
self.doneAdjusting = Observable()
class FitFunctionBase(object, metaclass=FitFunctionMeta):
expression = Expression()
name = 'None'
parameterNames = list()
def __init__(self):
numParameters = len(self.parameterNames)
self.epsfcn = 0.0
self.parameters = [0] * numParameters
self.startParameters = [1] * numParameters
self.startParameterExpressions = None # will be initialized by FitUiTableModel if values are available
self.parameterEnabled = [True] * numParameters
self.parametersConfidence = [None] * numParameters
self.units = None
self.results = SequenceDict({'RMSres': ResultRecord(name='RMSres')})
self.useSmartStartValues = False
self.hasSmartStart = not hasattr(self.smartStartValues, 'isNative')
self.parametersUpdated = Observable()
self.parameterBounds = [[None, None] for _ in range(numParameters)]
self.parameterBoundsExpressions = None
self.useErrorBars = True
def __setstate__(self, state):
state.pop('parameterNames', None)
state.pop('cov_x', None)
state.pop('infodict', None)
state.pop('laguerreCacheEta', None)
state.pop('laguerreTable', None)
state.pop('pnCacheBeta', None)
state.pop('pnTable', None)
self.__dict__ = state
self.__dict__.setdefault('useSmartStartValues', False)
self.__dict__.setdefault('startParameterExpressions', None)
self.__dict__.setdefault('parameterBounds',
[[None, None]
for _ in range(len(self.parameterNames))])
self.__dict__.setdefault('parameterBoundsExpressions', None)
self.__dict__.setdefault('useErrorBars', True)
self.hasSmartStart = not hasattr(self.smartStartValues, 'isNative')
def allFitParameters(self, p):
"""return a list where the disabled parameters are added to the enabled parameters given in p"""
pindex = 0
params = list()
for index, enabled in enumerate(self.parameterEnabled):
if enabled:
params.append(p[pindex])
pindex += 1
else:
params.append(float(self.startParameters[index]))
return params
@staticmethod
def coercedValue(val, bounds):
if bounds[1] is not None and val >= bounds[1]:
val = float(0.95 * bounds[1] + 0.05 *
bounds[0] if bounds[0] is not None else bounds[1] -
0.01)
if bounds[0] is not None and val <= bounds[0]:
val = float(0.95 * bounds[0] + 0.05 *
bounds[1] if bounds[1] is not None else bounds[0] +
0.01)
return val
def enabledStartParameters(self, parameters=None, bounded=False):
"""return a list of only the enabled start parameters"""
if parameters is None:
parameters = self.startParameters
params = list()
if bounded:
for enabled, param, bounds in zip(self.parameterEnabled,
parameters,
self.parameterBounds):
if enabled:
params.append(self.coercedValue(float(param), bounds))
else:
for enabled, param in zip(self.parameterEnabled, parameters):
if enabled:
params.append(float(param))
return params
def enabledFitParameters(self, parameters=None):
"""return a list of only the enabled fit parameters"""
if parameters is None:
parameters = self.parameters
params = list()
for enabled, param in zip(self.parameterEnabled, parameters):
if enabled:
params.append(float(param))
return params
def enabledParameterNames(self):
"""return a list of only the enabled fit parameters"""
params = list()
for enabled, param in zip(self.parameterEnabled, self.parameterNames):
if enabled:
params.append(param)
return params
def setEnabledFitParameters(self, parameters):
"""set the fitted parameters if enabled"""
pindex = 0
for index, enabled in enumerate(self.parameterEnabled):
if enabled:
self.parameters[index] = parameters[pindex]
pindex += 1
else:
self.parameters[index] = float(self.startParameters[index])
def setEnabledConfidenceParameters(self, confidence):
"""set the parameter confidence values for the enabled parameters"""
pindex = 0
for index, enabled in enumerate(self.parameterEnabled):
if enabled:
self.parametersConfidence[index] = confidence[pindex]
pindex += 1
else:
self.parametersConfidence[index] = None
@native
def smartStartValues(self, x, y, parameters, enabled):
return None
def enabledSmartStartValues(self, x, y, parameters):
smartParameters = self.smartStartValues(x, y, parameters,
self.parameterEnabled)
return [
smartparam if enabled else param for enabled, param, smartparam in
zip(self.parameterEnabled, parameters, smartParameters)
] if smartParameters is not None else None
def evaluate(self, globalDict):
myReplacementDict = self.replacementDict()
if globalDict is not None:
myReplacementDict.update(globalDict)
if self.startParameterExpressions is not None:
self.startParameters = [
param if expr is None else self.expression.evaluateAsMagnitude(
expr, myReplacementDict) for param, expr in zip(
self.startParameters, self.startParameterExpressions)
]
if self.parameterBoundsExpressions is not None:
self.parameterBounds = [[
bound[0]
if expr[0] is None else self.expression.evaluateAsMagnitude(
expr[0], myReplacementDict),
bound[1] if expr[1] is None else
self.expression.evaluateAsMagnitude(expr[0], myReplacementDict)
] for bound, expr in zip(self.parameterBounds,
self.parameterBoundsExpressions)]
def enabledBounds(self):
result = [[
float(bounds[0]) if bounds[0] is not None else None,
float(bounds[1]) if bounds[1] is not None else None
] for enabled, bounds in zip(self.parameterEnabled,
self.parameterBounds) if enabled]
enabled = any((any(bounds) for bounds in result))
return result if enabled else None
def leastsq(self, x, y, parameters=None, sigma=None):
logger = logging.getLogger(__name__)
# Ensure all values of sigma or non zero by replacing with the minimum nonzero value
if sigma is not None and self.useErrorBars:
nonzerosigma = sigma[sigma > 0]
sigma[sigma == 0] = numpy.min(
nonzerosigma) if len(nonzerosigma) > 0 else 1.0
else:
sigma = None
if parameters is None:
parameters = [float(param) for param in self.startParameters]
if self.useSmartStartValues:
smartParameters = self.smartStartValues(x, y, parameters,
self.parameterEnabled)
if smartParameters is not None:
parameters = [
smartparam if enabled else param
for enabled, param, smartparam in zip(
self.parameterEnabled, parameters, smartParameters)
]
myEnabledBounds = self.enabledBounds()
if myEnabledBounds:
enabledOnlyParameters, cov_x, infodict, self.mesg, self.ier = leastsqbound(
self.residuals,
self.enabledStartParameters(parameters, bounded=True),
args=(y, x, sigma),
epsfcn=self.epsfcn,
full_output=True,
bounds=myEnabledBounds)
else:
enabledOnlyParameters, cov_x, infodict, self.mesg, self.ier = leastsq(
self.residuals,
self.enabledStartParameters(parameters),
args=(y, x, sigma),
epsfcn=self.epsfcn,
full_output=True)
self.setEnabledFitParameters(enabledOnlyParameters)
self.update(self.parameters)
logger.info("chisq {0}".format(sum(infodict["fvec"] *
infodict["fvec"])))
# calculate final chi square
self.chisq = sum(infodict["fvec"] * infodict["fvec"])
self.dof = max(len(x) - len(parameters), 1)
RMSres = Q(sqrt(self.chisq / self.dof))
RMSres.significantDigits = 3
self.results['RMSres'].value = RMSres
# chisq, sqrt(chisq/dof) agrees with gnuplot
logger.info("success {0} {1}".format(self.ier, self.mesg))
logger.info("Converged with chi squared {0}".format(self.chisq))
logger.info("degrees of freedom, dof {0}".format(self.dof))
logger.info(
"RMS of residuals (i.e. sqrt(chisq/dof)) {0}".format(RMSres))
logger.info("Reduced chisq (i.e. variance of residuals) {0}".format(
self.chisq / self.dof))
# uncertainties are calculated as per gnuplot, "fixing" the result
# for non unit values of the reduced chisq.
# values at min match gnuplot
enabledParameterNames = self.enabledParameterNames()
if cov_x is not None:
enabledOnlyParametersConfidence = numpy.sqrt(
numpy.diagonal(cov_x)) * sqrt(self.chisq / self.dof)
self.setEnabledConfidenceParameters(
enabledOnlyParametersConfidence)
logger.info("Fitted parameters at minimum, with 68% C.I.:")
for i, pmin in enumerate(enabledOnlyParameters):
logger.info(
"%2i %-10s %12f +/- %10f" %
(i, enabledParameterNames[i], pmin,
sqrt(max(cov_x[i, i], 0)) * sqrt(self.chisq / self.dof)))
logger.info("Correlation matrix")
# correlation matrix close to gnuplot
messagelist = [" "]
for i in range(len(enabledOnlyParameters)):
messagelist.append("%-10s" % (enabledParameterNames[i], ))
logger.info(" ".join(messagelist))
messagelist = []
for i in range(len(enabledOnlyParameters)):
messagelist.append("%10s" % enabledParameterNames[i])
for j in range(i + 1):
messagelist.append(
"%10f" %
(cov_x[i, j] / sqrt(abs(cov_x[i, i] * cov_x[j, j])), ))
logger.info(" ".join(messagelist))
#-----------------------------------------------
else:
self.parametersConfidence = [None] * len(self.parametersConfidence)
return self.parameters
def __str__(self):
return "; ".join([
", ".join([self.name, self.functionString] + [
"{0}={1}".format(name, value)
for name, value in zip(self.parameterNames, self.parameters)
])
])
def setConstant(self, name, value):
setattr(self, name, value)
def update(self, parameters=None):
self.parametersUpdated.fire(values=self.replacementDict())
def toXmlElement(self, parent):
myroot = ElementTree.SubElement(parent, 'FitFunction', {
'name': self.name,
'functionString': self.functionString
})
for name, value, confidence, enabled, startExpression, bounds, boundsexpression in zip_longest(
self.parameterNames, self.parameters,
self.parametersConfidence, self.parameterEnabled,
self.startParameterExpressions, self.parameterBounds,
self.parameterBoundsExpressions):
e = ElementTree.SubElement(
myroot, 'Parameter', {
'name': name,
'confidence': repr(confidence),
'enabled': str(enabled),
'startExpression': str(startExpression),
'bounds': ",".join(map(str, bounds)),
'boundsExpression': ",".join(map(str, boundsexpression))
})
e.text = str(value)
for result in list(self.results.values()):
e = ElementTree.SubElement(myroot, 'Result', {
'name': result.name,
'definition': str(result.definition)
})
e.text = str(result.value)
return myroot
def toHdf5(self, group):
fitfunction_group = group.require_group('fitfunction')
fitfunction_group.attrs['name'] = self.name
fitfunction_group.attrs['functionString'] = self.functionString
parameter_group = fitfunction_group.require_group('parameters')
for index, (name, value, confidence, enabled, startExpression, bounds,
boundsexpression) in enumerate(
zip_longest(self.parameterNames, self.parameters,
self.parametersConfidence,
self.parameterEnabled,
self.startParameterExpressions,
self.parameterBounds,
self.parameterBoundsExpressions)):
g = parameter_group.require_group(name)
g.attrs['confidence'] = confidence
g.attrs['enabled'] = enabled
g.attrs['startExpression'] = str(startExpression)
g.attrs['bounds'] = bounds
g.attrs['boundsExpression'] = ",".join(map(str, boundsexpression))
g.attrs['value'] = value
g.attrs['index'] = index
results_group = fitfunction_group.require_group('results')
for result in list(self.results.values()):
g = results_group.requie_group(result.name)
g.attrs['definition'] = str(result.definition)
g.attrs['value'] = repr(result.value)
def residuals(self, p, y, x, sigma):
p = self.allFitParameters(p)
if sigma is not None:
return (y - self.functionEval(x, *p)) / sigma
else:
return y - self.functionEval(x, *p)
def value(self, x, p=None):
p = self.parameters if p is None else p
return self.functionEval(x, *p)
def replacementDict(self):
replacement = dict(list(zip(self.parameterNames, self.parameters)))
replacement.update(
dict(((v.name, v.value) for v in list(self.results.values()))))
return replacement
class LoadingHistory(object):
def __init__(self, dbConnection):
self.database_conn_str = dbConnection.connectionString
self.engine = create_engine(self.database_conn_str,
echo=dbConnection.echo)
self.isOpen = False
self.beginInsertRows = Observable()
self.endInsertRows = Observable()
self.beginResetModel = Observable()
self.endResetModel = Observable()
self._loadingEvents = list()
self._currentProfile = None
self._profiles = set()
def open(self):
try:
Base.metadata.create_all(self.engine)
self.Session = sessionmaker(bind=self.engine,
expire_on_commit=False)
self.session = self.Session()
self.isOpen = True
except OperationalError as e:
logging.getLogger(__name__).warning(
"Cannot open database: {0}".format(str(e)))
self.isOpen = False
def close(self):
self.session.commit()
self.isOpen = False
def __enter__(self):
if not self.isOpen:
self.open()
return self
def __exit__(self, exittype, value, tb):
self.session.commit()
def addLoadingEvent(self, loadingEvent):
try:
self.session.add(loadingEvent)
self.session.commit()
if self._currentProfile is None or loadingEvent.loadingProfile == self._currentProfile:
self.beginInsertRows.fire(first=len(self._loadingEvents),
last=len(self._loadingEvents))
self._loadingEvents.append(loadingEvent)
self.endInsertRows.firebare()
except (InvalidRequestError, IntegrityError, ProgrammingError) as e:
logging.getLogger(__name__).warning(str(e))
self.session.rollback()
self.session = self.Session()
def setTrappingDuration(self, duration):
self._loadingEvents[-1].trappingDuration = duration
self.commit()
def commit(self):
try:
self.session.commit()
except (InvalidRequestError, IntegrityError, ProgrammingError) as e:
logging.getLogger(__name__).warning(str(e))
self.session.rollback()
self.session = self.Session()
def query(self, fromTime, toTime, loadingProfile):
self.beginResetModel.firebare()
self._loadingEvents[:] = [] # clear the list in place
self._loadingEvents.extend(
self.session.query(LoadingEvent).
filter(LoadingEvent.trappingTime >= fromTime).filter(
LoadingEvent.trappingTime <= toTime).filter(
LoadingEvent.loadingProfile == loadingProfile).order_by(
LoadingEvent.trappingTime).all())
self.endResetModel.firebare()
self._currentProfile = loadingProfile
def getProfiles(self):
return None
@property
def loadingEvents(self):
return self._loadingEvents
def lastEvent(self):
return self._loadingEvents[-1]
class FitFunctionBase(object, metaclass=FitFunctionMeta):
expression = Expression()
name = 'None'
parameterNames = list()
def __init__(self):
numParameters = len(self.parameterNames)
self.epsfcn=0.0
self.parameters = [0] * numParameters
self.startParameters = [1] * numParameters
self.startParameterExpressions = None # will be initialized by FitUiTableModel if values are available
self.parameterEnabled = [True] * numParameters
self.parametersConfidence = [None] * numParameters
self.units = None
self.results = SequenceDict({'RMSres': ResultRecord(name='RMSres')})
self.useSmartStartValues = False
self.hasSmartStart = not hasattr(self.smartStartValues, 'isNative' )
self.parametersUpdated = Observable()
self.parameterBounds = [[None, None] for _ in range(numParameters) ]
self.parameterBoundsExpressions = None
self.useErrorBars = True
def __setstate__(self, state):
state.pop('parameterNames', None )
state.pop('cov_x', None)
state.pop('infodict', None)
state.pop('laguerreCacheEta', None )
state.pop('laguerreTable', None)
state.pop('pnCacheBeta', None )
state.pop('pnTable', None)
self.__dict__ = state
self.__dict__.setdefault( 'useSmartStartValues', False )
self.__dict__.setdefault( 'startParameterExpressions', None )
self.__dict__.setdefault( 'parameterBounds', [[None, None] for _ in range(len(self.parameterNames)) ] )
self.__dict__.setdefault( 'parameterBoundsExpressions', None)
self.__dict__.setdefault( 'useErrorBars', True)
self.hasSmartStart = not hasattr(self.smartStartValues, 'isNative' )
def allFitParameters(self, p):
"""return a list where the disabled parameters are added to the enabled parameters given in p"""
pindex = 0
params = list()
for index, enabled in enumerate(self.parameterEnabled):
if enabled:
params.append(p[pindex])
pindex += 1
else:
params.append(float(self.startParameters[index]))
return params
@staticmethod
def coercedValue( val, bounds ):
if bounds[1] is not None and val>=bounds[1]:
val = float(0.95*bounds[1]+0.05*bounds[0] if bounds[0] is not None else bounds[1]-0.01)
if bounds[0] is not None and val<=bounds[0]:
val = float(0.95*bounds[0]+0.05*bounds[1] if bounds[1] is not None else bounds[0]+0.01)
return val
def enabledStartParameters(self, parameters=None, bounded=False):
"""return a list of only the enabled start parameters"""
if parameters is None:
parameters = self.startParameters
params = list()
if bounded:
for enabled, param, bounds in zip(self.parameterEnabled, parameters, self.parameterBounds):
if enabled:
params.append(self.coercedValue(float(param), bounds))
else:
for enabled, param in zip(self.parameterEnabled, parameters):
if enabled:
params.append(float(param))
return params
def enabledFitParameters(self, parameters=None):
"""return a list of only the enabled fit parameters"""
if parameters is None:
parameters = self.parameters
params = list()
for enabled, param in zip(self.parameterEnabled, parameters):
if enabled:
params.append(float(param))
return params
def enabledParameterNames(self):
"""return a list of only the enabled fit parameters"""
params = list()
for enabled, param in zip(self.parameterEnabled, self.parameterNames):
if enabled:
params.append(param)
return params
def setEnabledFitParameters(self, parameters):
"""set the fitted parameters if enabled"""
pindex = 0
for index, enabled in enumerate(self.parameterEnabled):
if enabled:
self.parameters[index] = parameters[pindex]
pindex += 1
else:
self.parameters[index] = float(self.startParameters[index])
def setEnabledConfidenceParameters(self, confidence):
"""set the parameter confidence values for the enabled parameters"""
pindex = 0
for index, enabled in enumerate(self.parameterEnabled):
if enabled:
self.parametersConfidence[index] = confidence[pindex]
pindex += 1
else:
self.parametersConfidence[index] = None
@native
def smartStartValues(self, x, y, parameters, enabled):
return None
def enabledSmartStartValues(self, x, y, parameters):
smartParameters = self.smartStartValues(x, y, parameters, self.parameterEnabled)
return [ smartparam if enabled else param for enabled, param, smartparam in zip(self.parameterEnabled, parameters, smartParameters)] if smartParameters is not None else None
def evaluate(self, globalDict ):
myReplacementDict = self.replacementDict()
if globalDict is not None:
myReplacementDict.update( globalDict )
if self.startParameterExpressions is not None:
self.startParameters = [param if expr is None else self.expression.evaluateAsMagnitude(expr, myReplacementDict ) for param, expr in zip(self.startParameters, self.startParameterExpressions)]
if self.parameterBoundsExpressions is not None:
self.parameterBounds = [[bound[0] if expr[0] is None else self.expression.evaluateAsMagnitude(expr[0], myReplacementDict),
bound[1] if expr[1] is None else self.expression.evaluateAsMagnitude(expr[0], myReplacementDict)]
for bound, expr in zip(self.parameterBounds, self.parameterBoundsExpressions)]
def enabledBounds(self):
result = [[float(bounds[0]) if bounds[0] is not None else None,
float(bounds[1]) if bounds[1] is not None else None] for enabled, bounds in zip(self.parameterEnabled, self.parameterBounds) if enabled]
enabled = any( (any(bounds) for bounds in result) )
return result if enabled else None
def leastsq(self, x, y, parameters=None, sigma=None):
logger = logging.getLogger(__name__)
# Ensure all values of sigma or non zero by replacing with the minimum nonzero value
if sigma is not None and self.useErrorBars:
nonzerosigma = sigma[sigma>0]
sigma[sigma==0] = numpy.min(nonzerosigma) if len(nonzerosigma)>0 else 1.0
else:
sigma = None
if parameters is None:
parameters = [float(param) for param in self.startParameters]
if self.useSmartStartValues:
smartParameters = self.smartStartValues(x, y, parameters, self.parameterEnabled)
if smartParameters is not None:
parameters = [ smartparam if enabled else param for enabled, param, smartparam in zip(self.parameterEnabled, parameters, smartParameters)]
myEnabledBounds = self.enabledBounds()
if myEnabledBounds:
enabledOnlyParameters, cov_x, infodict, self.mesg, self.ier = leastsqbound(self.residuals, self.enabledStartParameters(parameters, bounded=True),
args=(y, x, sigma), epsfcn=self.epsfcn, full_output=True, bounds=myEnabledBounds)
else:
enabledOnlyParameters, cov_x, infodict, self.mesg, self.ier = leastsq(self.residuals, self.enabledStartParameters(parameters), args=(y, x, sigma),
epsfcn=self.epsfcn, full_output=True)
self.setEnabledFitParameters(enabledOnlyParameters)
self.update(self.parameters)
logger.info( "chisq {0}".format( sum(infodict["fvec"]*infodict["fvec"]) ) )
# calculate final chi square
self.chisq=sum(infodict["fvec"]*infodict["fvec"])
self.dof = max( len(x)-len(parameters), 1)
RMSres = Q(sqrt(self.chisq/self.dof))
RMSres.significantDigits = 3
self.results['RMSres'].value = RMSres
# chisq, sqrt(chisq/dof) agrees with gnuplot
logger.info( "success {0} {1}".format( self.ier, self.mesg ) )
logger.info( "Converged with chi squared {0}".format(self.chisq) )
logger.info( "degrees of freedom, dof {0}".format( self.dof ) )
logger.info( "RMS of residuals (i.e. sqrt(chisq/dof)) {0}".format( RMSres ) )
logger.info( "Reduced chisq (i.e. variance of residuals) {0}".format( self.chisq/self.dof ) )
# uncertainties are calculated as per gnuplot, "fixing" the result
# for non unit values of the reduced chisq.
# values at min match gnuplot
enabledParameterNames = self.enabledParameterNames()
if cov_x is not None:
enabledOnlyParametersConfidence = numpy.sqrt(numpy.diagonal(cov_x))*sqrt(self.chisq/self.dof)
self.setEnabledConfidenceParameters(enabledOnlyParametersConfidence)
logger.info( "Fitted parameters at minimum, with 68% C.I.:" )
for i, pmin in enumerate(enabledOnlyParameters):
logger.info( "%2i %-10s %12f +/- %10f"%(i, enabledParameterNames[i], pmin, sqrt(max(cov_x[i, i], 0))*sqrt(self.chisq/self.dof)) )
logger.info( "Correlation matrix" )
# correlation matrix close to gnuplot
messagelist = [" "]
for i in range(len(enabledOnlyParameters)): messagelist.append( "%-10s"%(enabledParameterNames[i],) )
logger.info( " ".join(messagelist))
messagelist = []
for i in range(len(enabledOnlyParameters)):
messagelist.append( "%10s"%enabledParameterNames[i] )
for j in range(i+1):
messagelist.append( "%10f"%(cov_x[i, j]/sqrt(abs(cov_x[i, i]*cov_x[j, j])),) )
logger.info( " ".join(messagelist))
#-----------------------------------------------
else:
self.parametersConfidence = [None]*len(self.parametersConfidence)
return self.parameters
def __str__(self):
return "; ".join([", ".join([self.name, self.functionString] + [ "{0}={1}".format(name, value) for name, value in zip(self.parameterNames, self.parameters)])])
def setConstant(self, name, value):
setattr(self, name, value)
def update(self,parameters=None):
self.parametersUpdated.fire( values=self.replacementDict() )
def toXmlElement(self, parent):
myroot = ElementTree.SubElement(parent, 'FitFunction', {'name': self.name, 'functionString': self.functionString})
for name, value, confidence, enabled, startExpression, bounds, boundsexpression in zip_longest(self.parameterNames, self.parameters, self.parametersConfidence, self.parameterEnabled, self.startParameterExpressions, self.parameterBounds, self.parameterBoundsExpressions):
e = ElementTree.SubElement( myroot, 'Parameter', {'name':name, 'confidence':repr(confidence), 'enabled': str(enabled), 'startExpression': str(startExpression), 'bounds': ",".join(map(str, bounds)),
'boundsExpression': ",".join(map(str, boundsexpression))})
e.text = str(value)
for result in list(self.results.values()):
e = ElementTree.SubElement( myroot, 'Result', {'name':result.name, 'definition':str(result.definition)})
e.text = str(result.value)
return myroot
def toHdf5(self, group):
fitfunction_group = group.require_group('fitfunction')
fitfunction_group.attrs['name'] = self.name
fitfunction_group.attrs['functionString'] = self.functionString
parameter_group = fitfunction_group.require_group('parameters')
for index, (name, value, confidence, enabled, startExpression, bounds, boundsexpression) in enumerate(zip_longest(self.parameterNames, self.parameters, self.parametersConfidence, self.parameterEnabled, self.startParameterExpressions, self.parameterBounds, self.parameterBoundsExpressions)):
g = parameter_group.require_group(name)
g.attrs['confidence'] = confidence
g.attrs['enabled'] = enabled
g.attrs['startExpression'] = str(startExpression)
g.attrs['bounds'] = bounds
g.attrs['boundsExpression'] = ",".join(map(str, boundsexpression))
g.attrs['value'] = value
g.attrs['index'] = index
results_group = fitfunction_group.require_group('results')
for result in list(self.results.values()):
g = results_group.requie_group(result.name)
g.attrs['definition'] = str(result.definition)
g.attrs['value'] = repr(result.value)
def residuals(self, p, y, x, sigma):
p = self.allFitParameters(p)
if sigma is not None:
return (y-self.functionEval(x, *p))/sigma
else:
return y-self.functionEval(x, *p)
def value(self,x,p=None):
p = self.parameters if p is None else p
return self.functionEval(x, *p )
def replacementDict(self):
replacement = dict(list(zip(self.parameterNames, self.parameters)))
replacement.update( dict( ( (v.name, v.value) for v in list(self.results.values()) ) ) )
return replacement
