def evaluate(self, stdResiduals:float=0.1, relError:float=0.1,
endTime:float=10.0, numPoint:int=30,
fractionParameterDeviation:float=0.5,
numIteration=NUM_BOOTSTRAP_ITERATION):
"""
Evaluates model fitting accuracy and bootstrapping for model
Parameters
----------
stdResiduals: Standard deviations of variable used in constructing reiduals
relError: relative error of parameter used in evaluation
endTime: ending time of the simulation
numPoint: number of points in the simulatin
fractionParameterDeviation: fractional amount that the parameter can vary
"""
# Construct synthetic observations
if self.selectedColumns is None:
data = self.roadRunner.simulate(0, endTime, numPoint)
else:
allColumns = list(self.selectedColumns)
if not TIME in allColumns:
allColumns.append(TIME)
data = self.roadRunner.simulate(0, endTime, numPoint, allColumns)
bracketTime = "[%s]" % TIME
if bracketTime in data.colnames:
# Exclude any column named '[time]'
idx = data.colnames.index(bracketTime)
dataArr = np.delete(data, idx, axis=1)
colnames = list(data.colnames)
colnames.remove(bracketTime)
colnames = [s[1:-1] if s != TIME else s for s in colnames]
simTS = NamedTimeseries(array=dataArr, colnames=colnames)
else:
simTS = NamedTimeseries(namedArray=data)
synthesizer = ObservationSynthesizerRandomErrors(
fittedTS=simTS, std=stdResiduals)
observedTS = synthesizer.calculate()
# Construct the parameter ranges
parameterDct = {}
for name in self.parameterValueDct.keys():
lower = self.parameterValueDct[name]*(1 - fractionParameterDeviation)
upper = self.parameterValueDct[name]*(1 + fractionParameterDeviation)
value = np.random.uniform(lower, upper)
parameterDct[name] = (lower, upper, value)
# Create the fitter
fitter = ModelFitter(self.roadRunner, observedTS,
selectedColumns=self.selectedColumns, parameterDct=parameterDct,
**self.kwargs)
msg = "Fitting the parameters %s" % str(self.parameterValueDct.keys())
self.logger.result(msg)
# Evaluate the fit
fitter.fitModel()
self._recordResult(fitter.params, relError, self.fitModelResult)
# Evaluate bootstrap
fitter.bootstrap(numIteration=numIteration)
if fitter.bootstrapResult is not None:
if fitter.bootstrapResult.numSimulation > 0:
self._recordResult(fitter.bootstrapResult.params,
relError, self.bootstrapResult)
def _init(self, numFold=3, numModel=NUM_MODEL):
"""
Initializes the test instance variables for a single fold.
Parameters
----------
numFold: int
numModel: int
"""
self.numModel = numModel
self.observedTS = NamedTimeseries(th.TEST_DATA_PATH)
numPoint = len(self.observedTS)
testIdxs = [n for n in range(numPoint) if n % numFold == 0]
trainIdxs = [n for n in range(numPoint) if n % numFold != 0]
self.modelNames = MODEL_NAMES[0:self.numModel]
self.trainTS = self.observedTS[trainIdxs]
self.testTS = self.observedTS[testIdxs]
self.modelSpecifications = mkRepeatedList(th.ANTIMONY_MODEL,
self.numModel)
self.trainTSCol = mkRepeatedList(self.testTS, self.numModel)
self.testTSDct = {n: self.testTS for n in self.modelNames}
self.parameterNames = list(th.PARAMETER_DCT.keys())
self.parameterNamesCollection = mkRepeatedList(self.parameterNames,
self.numModel)
self.suiteFitter = mkSuiteFitter(self.modelSpecifications,
self.trainTSCol,
self.parameterNamesCollection,
modelNames=self.modelNames)
self.wrapper = SuiteFitterWrapper(self.suiteFitter, self.testTSDct)
def testPlotSingle5(self):
if IGNORE_TEST:
return
timeseries = self.timeseries.subsetColumns(["S1"])
dct = {}
indices = [i for i in range(len(timeseries)) if i % 4 == 0]
for col in timeseries.allColnames:
dct[col] = timeseries[col][indices]
df = pd.DataFrame(dct)
meanTS = NamedTimeseries(dataframe=df)
meanTS[meanTS.colnames] = 1.1 * meanTS[meanTS.colnames]
stdTS = meanTS.copy()
for col in meanTS.colnames:
stdTS[col] = 1
#
self.plotter.plotTimeSingle(timeseries,
timeseries2=self.timeseries,
meanTS=meanTS,
stdTS=stdTS,
marker=[None, 'o', "^"],
color=["b", "r", "g"])
#
self.plotter.plotTimeSingle(timeseries, meanTS=meanTS, stdTS=stdTS)
#
self.plotter.plotTimeSingle(timeseries,
timeseries2=self.timeseries,
marker='*')
def mkTimeseries(length: int,
colnames: typing.List[str],
isRandom: bool = False) -> NamedTimeseries:
"""
Creates a time series of the desired shape.
Parameters
----------
length: number of time periods
colnames: names of the columns, excluding TIME
isRandom: generate uniform random numbers
Returns
-------
NamedTimeseries
"""
num_col = len(colnames)
if isRandom:
arr = np.random.random(num_col * length)
else:
arr = np.array(range(num_col * length))
matrix = np.reshape(arr, (length, len(colnames)))
timeseries = NamedTimeseries(array=matrix, colnames=colnames)
timeseries[TIME] = np.array(range(length))
return timeseries
def testCopyExisting(self):
if IGNORE_TEST:
return
timeseries = NamedTimeseries(timeseries=self.timeseries)
#
def checkVector(attribute):
length = len(self.timeseries.__getattribute__(attribute))
trues = [
timeseries.__getattribute__(attribute)[k] ==
self.timeseries.__getattribute__(attribute)[k]
for k in range(length)
]
self.assertTrue(all(trues))
def checkMatrix(attribute):
trues = []
for rowIdx, row in enumerate(
timeseries.__getattribute__(attribute)):
for colIdx, val in enumerate(row):
trues.append(val == self.timeseries.__getattribute__(
attribute)[rowIdx, colIdx])
self.assertTrue(all(trues))
#
for variable in ["start", "end"]:
self.assertEqual(timeseries.__getattribute__(variable),
self.timeseries.__getattribute__(variable))
for variable in ["colnames"]:
checkVector(variable)
for variable in ["values"]:
checkMatrix(variable)
def testWolfBug(self):
if IGNORE_TEST:
return
trueParameterDct = {
"J1_n": 4,
"J4_kp": 76411,
"J5_k": 80,
"J6_k": 9.7,
"J9_k": 28,
}
parametersToFit = [
SBstoat.Parameter("J1_n", lower=1, value=1, upper=8), # 4
SBstoat.Parameter("J4_kp", lower=3600, value=36000,
upper=150000), #76411
SBstoat.Parameter("J5_k", lower=10, value=10, upper=160), # 80
SBstoat.Parameter("J6_k", lower=1, value=1, upper=10), # 9.7
SBstoat.Parameter("J9_k", lower=1, value=50, upper=50), # 28
]
ts = NamedTimeseries(csvPath=WOLF_DATA)
methods = []
for optName in ["differential_evolution", "leastsq"]:
methods.append(SBstoat.OptimizerMethod(optName, {cn.MAX_NFEV: 10}))
fitter = ModelFitter(WOLF_MODEL,
ts,
parametersToFit=parametersToFit,
fitterMethods=methods)
fitter.fitModel()
for name in [p.name for p in parametersToFit]:
expected = trueParameterDct[name]
actual = fitter.params.valuesdict()[name]
self.assertLess(np.abs(np.log10(expected) - np.log10(actual)), 1.5)
self.assertTrue(name in fitter.reportFit())
def testConstructorNamedArray(self):
if IGNORE_TEST:
return
namedArray = self.model.simulate(0, 100, 30)
ts = NamedTimeseries(namedArray=namedArray)
self.assertTrue(
namedTimeseries.arrayEquals(namedArray.flatten(),
ts.values.flatten()))
def runSimulation(cls, **kwargs):
"""
Runs a simulation. Defaults to parameter values in the simulation.
Returns a NamedTimeseries.
Return
------
NamedTimeseries (or None if fail to converge)
"""
return NamedTimeseries(namedArray=cls.runSimulationNumpy(**kwargs))
def testToPandas(self):
if IGNORE_TEST:
return
df = self.timeseries.to_dataframe()
timeseries = NamedTimeseries(dataframe=df)
diff = set(df.columns).symmetric_difference(timeseries.colnames)
self.assertEqual(len(diff), 0)
total = sum(timeseries.values.flatten() -
self.timeseries.values.flatten())
self.assertTrue(np.isclose(total, 0))
def testConstructor2(self):
if IGNORE_TEST:
return
COLNAMES = [TIME, "S1", "S2"]
def test(timeseries, colnames=COLNAMES):
for name in colnames:
self.assertTrue(
np.isclose(sum(timeseries[name] - self.timeseries[name]),
0))
#
newTS = NamedTimeseries(colnames=COLNAMES,
array=self.timeseries[COLNAMES])
test(newTS)
# Check can use different cases for TIME
newTS = NamedTimeseries(colnames=["Time", "S1", "S2"],
array=self.timeseries[COLNAMES])
test(newTS)
def runSimulationDF(cls, **kwargs):
"""
Runs a simulation. Defaults to parameter values in the simulation.
Returns a NamedTimeseries.
Return
------
pd.DataFrame
"""
fittedTS = NamedTimeseries(namedArray=cls.runSimulationArr(**kwargs))
return fittedTS.to_dataframe()
def test(values, reference):
df = pd.DataFrame({TIME: TIMES, VALUE: values})
ts = NamedTimeseries(dataframe=df)
results = ts.getTimesForValue(VALUE, reference)
for time in results:
idx1 = int(time)
idx2 = idx1 + 1
small = min(ts[VALUE][idx1], ts[VALUE][idx2])
large = max(ts[VALUE][idx1], ts[VALUE][idx2])
self.assertLessEqual(small, reference)
self.assertGreaterEqual(large, reference)
def _mkTimeseries(numRow=NUM_ROW, numCol=NUM_COL, std=STD):
colnames = ["V%d" % d for d in range(NUM_COL)]
allColnames = list(colnames)
allColnames.insert(0, TIME)
timeArr = np.array(range(NUM_ROW))
timeArr = np.reshape(timeArr, (NUM_ROW, 1))
#
def addTime(arr):
newArr = np.concatenate([timeArr, arr], axis=1)
return newArr
#
residualsTS = NamedTimeseries(array=addTime(
np.random.normal(0, std, (numRow, numCol))),
colnames=allColnames)
fittedTS = NamedTimeseries(array=addTime(
np.random.normal(MEAN, 10 * std, (numRow, numCol))),
colnames=allColnames)
fittedTS[colnames] = np.floor(fittedTS[colnames])
observedTS = fittedTS.copy()
observedTS[colnames] += residualsTS[colnames]
return observedTS, fittedTS, residualsTS
def _init(self, numModel=NUM_MODEL):
self.numModel = numModel
self.observedTS = NamedTimeseries(th.TEST_DATA_PATH)
self.modelNames = MODEL_NAMES[0:numModel]
self.modelSpecifications = mkRepeatedList(th.ANTIMONY_MODEL, numModel)
self.datasets = mkRepeatedList(self.observedTS, numModel)
self.parameterNames = list(th.PARAMETER_DCT.keys())
self.parameterNamesCollection = mkRepeatedList(self.parameterNames,
numModel)
self.suiteFitter = mkSuiteFitter(self.modelSpecifications,
self.datasets,
self.parameterNamesCollection,
modelNames=self.modelNames,
fitterMethods=METHODS)
def mkDataSourceDct(filePath,
colName,
dataSourceNames=None,
isTimeColumns=True):
"""
Creates a dataSource dictionary as required by ModelStudy from
a file whose columns are observed values of the same variable.
Parameters
----------
filepath: str
path to the file containing columns of observed values
colName: str
Name of the simulation variable to fit to observed values
dataSourceNames: list-str
Names for the instances of observedValues
if None, then column headers are used (or column number)
isTimeColumns: boolean
Columns are time. If False, rows are time.
Returns
-------
dict: key is instance name; value is NamedTimeseries
"""
dataDF = pd.read_csv(filePath, header=None)
if isTimeColumns:
dataDF = dataDF.transpose()
if dataSourceNames is not None:
# Use instance names
if len(dataSourceNames) != len(dataDF.columns):
msg = "Number of instances is not equal to the number of columns"
raise ValueError(msg)
dct = {k: v for k, v in zip(dataDF.columns, dataSourceNames)}
dataDF = dataDF.rename(columns=dct)
# Ensure that column names are strings
dct = {k: str(k) for k in dataDF.columns}
dataDF = dataDF.rename(columns=dct)
dataDF.index.name = "time"
# Construct the data source dictionary
dataDF.index = range(len(dataDF))
dataDF.index.name = TIME
dataTS = NamedTimeseries(dataframe=dataDF)
dataSourceDct = {d: dataTS.subsetColumns([d]) for d in dataDF.columns}
dataSourceDct = {
d: dataSourceDct[d].rename(d, colName)
for d in dataSourceDct.keys()
}
#
return dataSourceDct
def simulate(self, **kwargs):
"""
Runs a simulation. Defaults to parameter values in the simulation.
Parameters
----------
params: lmfit.Parameters
startTime: float
endTime: float
numPoint: int
Return
------
NamedTimeseries
"""
fixedArr = self._simulateNumpy(**kwargs)
return NamedTimeseries(namedArray=fixedArr)
def _transformFittedTS(self, data):
"""
Updates the fittedTS taking into account required transformations.
Parameters
----------
data: np.ndarray
Results
----------
NamedTimeseries
"""
colnames = list(self.selectedColumns)
colnames.insert(0, TIME)
fittedTS = NamedTimeseries(array=data[:, :], colnames=colnames)
if self.fittedDataTransformDct is not None:
for column, func in self.fittedDataTransformDct.items():
if func is not None:
fittedTS[column] = func(fittedTS)
return fittedTS
def testExamples(self):
if IGNORE_TEST:
return
# Create from file
timeseries = NamedTimeseries(csvPath=TEST_DATA_PATH)
# NamedTimeseries can use len function
length = len(timeseries) # number of rows
# Extract the numpy array values using indexing
timeValues = timeseries["time"]
s1Values = timeseries["S1"]
# Get the start and end times
startTime = timeseries.start
endTime = timeseries.end
# Create a new time series that subsets the variables of the old one
colnames = ["time", "S1", "S2"]
newTS = mkNamedTimeseries(colnames, timeseries[colnames])
# Create a new timeseries that excludes time 0
ts2 = timeseries[1:]
# Create a new column variable
timeseries["S8"] = timeseries["time"]**2 + 3 * timeseries["S1"]
timeseries["S9"] = 10 # Assign a constant to all rows
def testWolfBug(self):
if IGNORE_TEST:
return
fullDct = {
#"J1_n": (1, 1, 8), # 4
#"J4_kp": (3600, 36000, 150000), #76411
#"J5_k": (10, 10, 160), # 80
#"J6_k": (1, 1, 10), # 9.7
"J9_k": (1, 50, 50), # 28
}
for parameter in fullDct.keys():
logger = Logger(logLevel=LEVEL_MAX)
logger = Logger()
ts = NamedTimeseries(csvPath=WOLF_DATA)
parameterDct = {parameter: fullDct[parameter]}
fitter = ModelFitter(WOLF_MODEL, ts[0:100],
parameterDct=parameterDct,
logger=logger, fitterMethods=[
"differential_evolution", "leastsq"])
fitter.fitModel()
self.assertTrue("J9_k" in fitter.reportFit())
def copy(self, isKeepLogger=False, isKeepOptimizer=False, **kwargs):
"""
Creates a copy of the model fitter, overridding any argument
that is not None.
Preserves the user-specified settings and the results
of bootstrapping.
Parameters
----------
isKeepLogger: bool
isKeepOptimizer: bool
kwargs: dict
arguments to override in copy
Returns
-------
ModelFitter
"""
def setValues(names):
"""
Sets the value for a list of names.
Parameters
----------
names: list-str
Returns
-------
list-object
"""
results = []
for name in names:
altName = "_%s" % name
if name in kwargs.keys():
results.append(kwargs[name])
elif name in self.__dict__.keys():
results.append(self.__dict__[name])
elif altName in self.__dict__.keys():
results.append(self.__dict__[altName])
else:
raise RuntimeError("%s: not found" % name)
return results
#
# Get the positional and keyword arguments
inspectResult = inspect.getfullargspec(ModelFitterCore.__init__)
allArgNames = inspectResult.args[1:] # Ignore 'self'
if inspectResult.defaults is None:
numKwarg = 0
else:
numKwarg = len(inspectResult.defaults)
numParg = len(allArgNames) - numKwarg # positional arguments
pargNames = allArgNames[:numParg]
kwargNames = allArgNames[numParg:]
# Construct the arguments
callPargs = setValues(pargNames)
callKwargValues = setValues(kwargNames)
callKwargs = {n: v for n, v in zip(kwargNames, callKwargValues)}
if numKwarg > 0:
# Adjust model specification
modelSpecificationIdx = pargNames.index("modelSpecification")
observedDataIdx = pargNames.index("observedData")
modelSpecification = callPargs[modelSpecificationIdx]
observedTS = NamedTimeseries(callPargs[observedDataIdx])
selectedColumns = callKwargs["selectedColumns"]
if not isinstance(modelSpecification, str):
try:
modelSpecification = self.modelSpecification.getAntimony()
callPargs[modelSpecificationIdx] = modelSpecification
observedTS, selectedColumns = self._adjustNames(
modelSpecification, observedTS)
callPargs[observedDataIdx] = observedTS
callKwargs["selectedColumns"] = selectedColumns
except Exception as err:
self.logger.error(
"Problem wth conversion to Antimony. Details:", err)
raise ValueError("Cannot proceed.")
#
if isKeepLogger:
callKwargs["logger"] = self.logger
if numParg < 2:
callPargs = [None, None]
newModelFitter = self.__class__(*callPargs, **callKwargs)
if self.optimizer is not None:
if isKeepOptimizer:
newModelFitter.optimizer = self.optimizer.copyResults()
if self.bootstrapResult is not None:
newModelFitter.bootstrapResult = self.bootstrapResult.copy()
return newModelFitter
def getTimeseries():
return NamedTimeseries(TEST_DATA_PATH)
def runSimulation(
cls,
parameters=None,
roadrunner=None,
startTime=0,
endTime=5,
numPoint=30,
selectedColumns=None,
returnDataFrame=True,
_logger=Logger(),
_loggerPrefix="",
):
"""
Runs a simulation. Defaults to parameter values in the simulation.
Parameters
----------
roadrunner: ExtendedRoadRunner/str
Roadrunner model
parameters: lmfit.Parameters
lmfit parameters
startTime: float
start time for the simulation
endTime: float
end time for the simulation
numPoint: int
number of points in the simulation
selectedColumns: list-str
output columns in simulation
returnDataFrame: bool
return a DataFrame
_logger: Logger
_loggerPrefix: str
Return
------
NamedTimeseries (or None if fail to converge)
"""
if isinstance(roadrunner, str):
roadrunner = cls.initializeRoadrunnerModel(roadrunner)
else:
roadrunner.reset()
if parameters is not None:
# Parameters have been specified
cls.setupModel(roadrunner, parameters, logger=_logger)
# Do the simulation
if selectedColumns is not None:
newSelectedColumns = list(selectedColumns)
if TIME not in newSelectedColumns:
newSelectedColumns.insert(0, TIME)
try:
data = roadrunner.simulate(startTime, endTime, numPoint,
newSelectedColumns)
except Exception as err:
_logger.error("Roadrunner exception: ", err)
data = None
else:
try:
data = roadrunner.simulate(startTime, endTime, numPoint)
except Exception as err:
_logger.exception("Roadrunner exception: %s", err)
data = None
if data is None:
return data
fittedTS = NamedTimeseries(namedArray=data)
if returnDataFrame:
result = fittedTS.to_dataframe()
else:
result = fittedTS
return result
def setUp(self):
self.timeseries = NamedTimeseries(csvPath=TEST_DATA_PATH)
self.model = te.loada(ANTIMONY_MODEL)
def simulate(self,
params=None,
startTime=None,
endTime=None,
numPoint=None):
"""
Runs a simulation. Defaults to parameter values in the simulation.
Parameters
----------
params: lmfit.Parameters
startTime: float
endTime: float
numPoint: int
Return
------
NamedTimeseries
"""
def set(default, parameter):
# Sets to default if parameter unspecified
if parameter is None:
return default
else:
return parameter
##V
block = Logger.join(self._loggerPrefix, "fitModel.simulate")
guid = self.logger.startBlock(block)
## V
sub1Block = Logger.join(block, "sub1")
sub1Guid = self.logger.startBlock(sub1Block)
startTime = set(self.observedTS.start, startTime)
endTime = set(self.observedTS.end, endTime)
numPoint = set(len(self.observedTS), numPoint)
## V
sub1aBlock = Logger.join(sub1Block, "sub1a")
sub1aGuid = self.logger.startBlock(sub1aBlock)
if self.roadrunnerModel is None:
self._initializeRoadrunnerModel()
self.roadrunnerModel.reset()
## ^
self.logger.endBlock(sub1aGuid)
## V
sub1bBlock = Logger.join(sub1Block, "sub1b")
sub1bGuid = self.logger.startBlock(sub1bBlock)
if params is not None:
# Parameters have been specified
self._setupModel(params)
## ^
self.logger.endBlock(sub1bGuid)
# Do the simulation
selectedColumns = list(self.selectedColumns)
if not TIME in selectedColumns:
selectedColumns.insert(0, TIME)
## ^
self.logger.endBlock(sub1Guid)
## V
roadrunnerBlock = Logger.join(block, "roadrunner")
roadrunnerGuid = self.logger.startBlock(roadrunnerBlock)
data = self.roadrunnerModel.simulate(startTime, endTime, numPoint,
selectedColumns)
self.logger.endBlock(roadrunnerGuid)
## ^
# Select the required columns
## V
sub2Block = Logger.join(block, "sub2")
sub2Guid = self.logger.startBlock(sub2Block)
fittedTS = NamedTimeseries(namedArray=data)
self.logger.endBlock(sub2Guid)
## ^
self.logger.endBlock(guid)
##^
return fittedTS
def mkTimeSeries(self, values, name):
numPoint = len(values)
arr = np.array([range(numPoint), values])
arr = np.resize(arr, (2, numPoint))
arr = arr.transpose()
return NamedTimeseries(array=arr, colnames=["time", name])
def testMissingData(self):
if IGNORE_TEST:
return
with self.assertRaises(ValueError):
timeseries = NamedTimeseries(csvPath=TEST_BAD_DATA_PATH)
def getTS(data):
if isinstance(data, pd.DataFrame):
return NamedTimeseries(dataframe=data)
return data
def _mkParameterDF(self, parameters=None):
df = pd.DataFrame(self.bootstrapResult.parameterDct)
if parameters is not None:
df = df[parameters]
df.index.name = TIME
return NamedTimeseries(dataframe=df)
def setUp(self):
self.timeseries = NamedTimeseries(csvPath=TEST_DATA_PATH)
self.plotter = TimeseriesPlotter(isPlot=IS_PLOT)
def getTimeseries():
return NamedTimeseries(csvPath=TEST_DATA_PATH)