def createRunners(self):
# move the current runners, if any, to the array with old runners
if self.runners:
self.oldCpuTimes.append(max(r.currentCpuTime for r in self.runners))
self.oldRunners.extend(self.runners)
self.runners = []
# reset other state
self.convergenceTime = None
self.lastOfUpdateTime = time.time()
self.runnerGeneration += 1
bestParams = None
if self.oldRunners and bool(g.getConfig("optimization.restartFromBestValue")):
# get best params
bestParams = self.getBestParams()
for id in range(int(g.getConfig("optimization.runsPerJob"))):
r = runner.Runner(self, id + 1, self.currentMethod, self.runnerGeneration)
if not r.prepare(self.workDir, self.copasiFile, bestParams):
g.log(LOG_ERROR, "{}: failed to create a runner".format(r.getName()))
return False
self.runners.append(r)
# note that this may create more processes than the number of free CPU cores!
for r in self.runners:
r.execute()
return True
def __init__(self, pool, params, maxCores, areParametersChangeable):
self.pool = pool
self.params = params
self.areParametersChangeable = areParametersChangeable
self.id = pool.strategy.nextJobID
pool.strategy.nextJobID += 1
self.methods = copy.copy(g.getConfig("copasi.methods"))
assert len(self.methods)
self.fallbackMethods = copy.copy(g.getConfig("copasi.fallbackMethods"))
self.currentMethod = None
self.pastMethods = []
self.isUsingFallback = False
self.runners = []
self.oldRunners = []
self.runnerGeneration = 0
self.oldCpuTimes = []
self.convergenceTime = None
self.convergenceValue = None
self.copasiFile = None
self.workDir = None
self.startTime = time.time()
self.lastOfUpdateTime = self.startTime
self.lastBalanceTime = self.startTime - LOAD_BALANCE_INTERVAL
# XXX: note that the number of free cores can increase during job's
# lifetime, but the code will not react to that, keeping maxCores constant
self.maxCores = maxCores
self.loadBalanceFactor = float(g.getConfig("optimization.runsPerJob")) / maxCores
def createRunners(self):
# move the current runners, if any, to the array with old runners
if self.runners:
self.oldCpuTimes.append(max(r.currentCpuTime for r in self.runners))
self.oldRunners.extend(self.runners)
self.runners = []
# reset other state
self.convergenceTime = None
self.lastOfUpdateTime = time.time()
self.runnerGeneration += 1
bestParams = None
if self.oldRunners and bool(g.getConfig("optimization.restartFromBestValue")):
# get best params
bestParams = self.getBestParams()
for id in range(int(g.getConfig("optimization.runsPerJob"))):
r = runner.Runner(self, id + 1, self.currentMethod, self.runnerGeneration)
if not r.prepare(self.workDir, self.copasiFile, bestParams):
g.log(LOG_ERROR, "{}: failed to create a runner".format(r.getName()))
return False
self.runners.append(r)
# note that this may create more processes than the number of free CPU cores!
for r in self.runners:
r.execute()
return True
def __init__(self, pool, params, maxCores, areParametersChangeable):
self.pool = pool
self.params = params
self.areParametersChangeable = areParametersChangeable
self.id = pool.strategy.nextJobID.next()
self.methods = copy.copy(g.getConfig("copasi.methods"))
assert len(self.methods)
self.fallbackMethods = copy.copy(g.getConfig("copasi.fallbackMethods"))
self.currentMethod = None
self.isUsingFallback = False
self.runners = []
self.oldRunners = []
self.runnerGeneration = 0
self.oldCpuTimes = []
self.convergenceTime = None
self.convergenceValue = None
self.copasiFile = None
self.workDir = None
self.startTime = time.time()
self.lastOfUpdateTime = self.startTime
self.lastBalanceTime = self.startTime - LOAD_BALANCE_INTERVAL
# XXX: note that the number of free cores can increase during job's
# lifetime, but the code will not react to that, keeping maxCores constant
self.maxCores = maxCores
self.loadBalanceFactor = float(g.getConfig("optimization.runsPerJob")) / maxCores
def totalOptimizationPotentialReached(self, numParameters):
if not self.isTOPEnabled():
return False
targetFraction = g.getConfig("optimization.targetFractionOfTOP")
# calculate the target value, looking at both config and at the job with all parameters, if any
try:
configTarget = float(g.getConfig("optimization.bestOfValue"))
except:
g.log(
LOG_INFO, "Bad bestOfValue in config: {}".format(
g.getConfig("optimization.bestOfValue")))
return False
joblist = self.jobsByBestOfValue[-1]
if joblist:
calculatedTarget = joblist[0].getBestOfValue()
else:
calculatedTarget = MIN_OF_VALUE
targetValue = max(configTarget, calculatedTarget)
if targetValue == MIN_OF_VALUE:
g.log(
LOG_DEBUG,
"TOP: no target value: {} {}".format(configTarget,
calculatedTarget))
return False
achievedValue = MIN_OF_VALUE
for joblist in self.jobsByBestOfValue[:numParameters + 1]:
if joblist:
achievedValue = max(achievedValue, joblist[0].getBestOfValue())
isReached = False
if self.topBaseline > targetValue:
isReached = True
requiredValue = targetValue
else:
requiredValue = (targetValue - self.topBaseline
) * targetFraction + self.topBaseline
isReached = achievedValue >= requiredValue
g.log(
LOG_DEBUG,
"TOP: {} parameters, {} achieved, {} required, {} target, {} configTarget, {} calculatedTarget"
.format(numParameters, achievedValue, requiredValue, targetValue,
configTarget, calculatedTarget))
if isReached:
g.log(
LOG_INFO,
"Terminating optimization at {} parameters: good-enough-value criteria reached (required {})"
.format(numParameters, requiredValue))
return True
return False
def dumpResults(self, totalLimit=0, perParamLimit=0):
if g.workDir is None:
return None
filename = g.getConfig("output.filename")
# do not allow put the results in other directories because of security reasons
if filename != os.path.basename(filename):
g.log(
LOG_INFO,
"output file name should not include path, ignoring all but the last element in it"
)
filename = os.path.basename(filename)
(name, ext) = os.path.splitext(filename)
if not ext:
ext = ".csv" # default
filename = os.path.join(g.workDir,
"{}-{}{}".format(name, "-", g.taskName, ext))
with self.jobLock:
if len(self.finishedJobs) <= self.lastNumJobsDumped:
# all finished jobs already were saved, nothing to do
return filename
self.lastNumJobsDumped = len(self.finishedJobs)
allJobsByBestOfValue = []
if perParamLimit == 0:
numberOfBestCombinations = int(
g.getConfig("output.numberOfBestCombinations"))
else:
numberOfBestCombinations = perParamLimit
for joblist in self.jobsByBestOfValue:
if numberOfBestCombinations:
lst = joblist[:numberOfBestCombinations]
else:
lst = joblist
for job in lst:
allJobsByBestOfValue.append(job)
allJobsByBestOfValue.sort(key=lambda x: x.getBestOfValue(),
reverse=True)
allParams = self.copasiConfig["params"]
cnt = 0
with open(filename, "w") as f:
self.dumpCsvFileHeader(f)
for job in allJobsByBestOfValue:
job.dumpResults(f, allParams)
cnt += 1
if totalLimit and cnt >= totalLimit:
break
g.log(
LOG_INFO, '<spacescanner>: results of finished jobs saved in "' +
filename + '"')
return filename
def __init__(self, strategy, parameterSets, areParametersChangeable):
self.strategy = strategy
self.parameterSets = parameterSets
self.areParametersChangeable = areParametersChangeable
self.currentParametersIndex = 0
self.jobLock = threading.Lock()
self.activeJobs = []
self.numUsableCores = max(1, int(g.getConfig("optimization.maxConcurrentRuns")))
self.numRunnersPerJob = max(1, int(g.getConfig("optimization.runsPerJob")))
self.maxNumParallelJobs = int(math.ceil(float(self.numUsableCores) / self.numRunnersPerJob))
self.bestOfValue = MIN_OF_VALUE
self.bestParams = []
def __init__(self, strategy, parameterSets, areParametersChangeable):
self.strategy = strategy
self.parameterSets = parameterSets
self.areParametersChangeable = areParametersChangeable
self.currentParametersIndex = 0
self.jobLock = threading.Lock()
self.activeJobs = []
self.numUsableCores = max(1, int(g.getConfig("optimization.maxConcurrentRuns")))
self.numRunnersPerJob = max(1, int(g.getConfig("optimization.runsPerJob")))
self.maxNumParallelJobs = int(math.ceil(float(self.numUsableCores) / self.numRunnersPerJob))
self.bestOfValue = MIN_OF_VALUE
self.bestParams = []
def isTOPEnabled(self):
targetFraction = g.getConfig("optimization.targetFractionOfTOP")
if targetFraction is None or targetFraction == 0.0:
# TOP is disabled
return False
# TOP is enabled
return True
def isTOPEnabled(self):
optimality = g.getConfig("optimization.optimalityRelativeError")
if optimality is None or optimality == 0.0:
# TOP is disabled
return False
# TOP is enabled
return True
def executeCopasi(runner):
# a pseudo-loop for simpler error handling
while True:
if bool(g.getConfig("webTestMode")):
if runner.job.id < 4:
time.sleep(1.0)
runner.ofValue = random.random() * 100
runner.isActive = False
else:
# wait for the strategy to quit
while not runner.job.pool.strategy.doQuitFlag:
time.sleep(1.0)
runner.ofValue += random.random()
runner.isActive = False
break
g.log(LOG_DEBUG, "executing " + " ".join(runner.process.args))
runner.isError = runner.process.run()
# check report in order to update OF value (even if nonzero return value)
with reportLock:
runner.checkReport(hasTerminated=True, now=time.time())
# exit the loop without an error
break
# check termination conditions at the global run level
g.log(LOG_DEBUG, "{}: terminated".format(runner.getName()))
runner.isActive = False
# disable this; do it from polling in the main thread instead
#runner.job.checkIfHasTerminated()
# overwrite the .cps file with best parameter values
runner.cleanup()
def executeCopasi(runner):
# a pseudo-loop for simpler error handling
while True:
if bool(g.getConfig("webTestMode")):
if runner.job.id < 4:
time.sleep(1.0)
runner.ofValue = random.random() * 100
runner.isActive = False
else:
# wait for the strategy to quit
while not runner.job.pool.strategy.doQuitFlag:
time.sleep(1.0)
runner.ofValue += random.random()
runner.isActive = False
break
g.log(LOG_DEBUG, "executing " + " ".join(runner.process.args))
runner.isError = runner.process.run()
# check report in order to update OF value (even if nonzero return value)
with reportLock:
runner.checkReport(hasTerminated=True, now=time.time())
# exit the loop without an error
break
# check termination conditions at the global run level
g.log(LOG_DEBUG, "{}: terminated".format(runner.getName()))
runner.isActive = False
# disable this; do it from polling in the main thread instead
# runner.job.checkIfHasTerminated()
# overwrite the .cps file with best parameter values
runner.cleanup()
def loadCopasiFile(self):
copasiFile = copasifile.CopasiFile()
filename = g.getConfig("copasi.modelFile")
filename = filename.replace("@SELF@", SELF_PATH)
g.log(LOG_INFO, "<spacescanner>: opening COPASI model file {}".format(filename))
if not copasiFile.read(filename):
return None
return copasiFile
def dumpResults(self, totalLimit = 0, perParamLimit = 0):
if g.workDir is None:
return None
filename = g.getConfig("output.filename")
# do not allow put the results in other directories because of security reasons
if filename != os.path.basename(filename):
g.log(LOG_INFO, "output file name should not include path, ignoring all but the last element in it")
filename = os.path.basename(filename)
(name, ext) = os.path.splitext(filename)
if not ext:
ext = ".csv" # default
filename = os.path.join(g.workDir, "{}-{}{}".format(name, "-", g.taskName, ext))
with self.jobLock:
if len(self.finishedJobs) <= self.lastNumJobsDumped:
# all finished jobs already were saved, nothing to do
return filename
self.lastNumJobsDumped = len(self.finishedJobs)
allJobsByBestOfValue = []
if perParamLimit == 0:
numberOfBestCombinations = int(g.getConfig("output.numberOfBestCombinations"))
else:
numberOfBestCombinations = perParamLimit
for joblist in self.jobsByBestOfValue:
if numberOfBestCombinations:
lst = joblist[:numberOfBestCombinations]
else:
lst = joblist
for job in lst:
allJobsByBestOfValue.append(job)
allJobsByBestOfValue.sort(key=lambda x: x.getBestOfValue(), reverse=True)
allParams = self.copasiConfig["params"]
cnt = 0
with open(filename, "w") as f:
self.dumpCsvFileHeader(f)
for job in allJobsByBestOfValue:
job.dumpResults(f, allParams)
cnt += 1
if totalLimit and cnt >= totalLimit:
break
g.log(LOG_INFO, '<spacescanner>: results of finished jobs saved in "' + filename + '"')
return filename
def loadCopasiFile(self):
copasiFile = copasifile.CopasiFile()
filename = g.getConfig("copasi.modelFile")
filename = filename.replace("@SELF@", SELF_PATH)
g.log(LOG_INFO,
"<spacescanner>: opening COPASI model file {}".format(filename))
if not copasiFile.read(filename):
return None
return copasiFile
def hasConsensus(self):
epsilonAbs = float(g.getConfig("optimization.consensusAbsoluteError"))
epsilonRel = float(g.getConfig("optimization.consensusRelativeError"))
if self.convergenceTime is None:
minV = min([r.ofValue for r in self.runners])
else:
minV = self.convergenceValue
# if this is not the first method, also should use max from the previous
maxV = self.getBestOfValue()
# returns true if either the absolute difference OR relative difference are small
if floatEqual(minV, maxV, epsilonAbs):
return True
# XXX: avoid division by zero; this means relative convergence will always fail on 0.0
if math.isinf(minV) or math.isinf(maxV) or maxV == 0.0:
return False
return abs(1.0 - minV / maxV) < epsilonRel
def ioGetActiveJobs(self, qs):
response = []
with self.jobLock:
if bool(g.getConfig("webTestMode")):
for id in self.finishedJobs:
if id < 4:
response.append(self.finishedJobs[id].getStatsFull())
if self.activeJobPool:
with self.activeJobPool.jobLock:
for job in self.activeJobPool.activeJobs:
response.append(job.getStatsFull())
return response
def execute(self, workDir, copasiFile):
self.workDir = workDir
self.copasiFile = copasiFile
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.methods)
else:
self.currentMethod = self.methods[0]
g.log(LOG_INFO, "starting " + self.getFullName())
return self.createRunners()
def execute(self, workDir, copasiFile):
self.workDir = workDir
self.copasiFile = copasiFile
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.methods)
else:
self.currentMethod = self.methods[0]
g.log(LOG_INFO, "starting " + self.getFullName())
return self.createRunners()
def totalOptimizationPotentialReached(self, numParameters):
if not self.isTOPEnabled():
return False
optimality = g.getConfig("optimization.optimalityRelativeError")
# calculate the target value, looking at both config and at the job with all parameters, if any
try:
configTarget = float(g.getConfig("optimization.bestOfValue"))
except:
g.log(LOG_INFO, "Bad bestOfValue in config: {}".format(g.getConfig("optimization.bestOfValue")))
return False
joblist = self.jobsByBestOfValue[-1]
if joblist:
calculatedTarget = joblist[0].getBestOfValue()
else:
calculatedTarget = MIN_OF_VALUE
targetValue = max(configTarget, calculatedTarget)
if targetValue == MIN_OF_VALUE:
return False
achievedValue = MIN_OF_VALUE
for joblist in self.jobsByBestOfValue[:numParameters + 1]:
if joblist:
achievedValue = max(achievedValue, joblist[0].getBestOfValue())
proportion = 1.0 - float(optimality)
isReached = False
if self.topBaseline > targetValue:
isReached = True
requiredValue = targetValue
else:
requiredValue = (targetValue - self.topBaseline) * proportion + self.topBaseline
isReached = achievedValue >= requiredValue
if isReached:
g.log(LOG_INFO, "terminating optimization at {} parameters: good-enough-value criteria reached (required {})".format(numParameters, requiredValue))
return True
return False
def getAllStats(self):
if bool(g.getConfig("webTestMode")):
r = 0.0
t = 0.0
random.seed(0)
result = []
for i in range(20):
r += random.random()
t += random.random()
si = StatsItem("{} {} 1 ( 74.24 ) -1.81914".format(r, t))
result.append(si)
return result
return self.getAllStatsForGeneration(self.generation)
def getAllStats(self):
if bool(g.getConfig("webTestMode")):
r = 0.0
t = 0.0
random.seed(0)
result = []
for i in range(20):
r += random.random()
t += random.random()
si = StatsItem("{} {} 1 ( 74.24 ) -1.81914".format(r, t))
result.append(si)
return result
return self.getAllStatsForGeneration(self.generation)
def hasConsensus(self):
epsilonAbs = float(g.getConfig("optimization.consensusAbsoluteError"))
epsilonRel = float(g.getConfig("optimization.consensusCorridor"))
if self.convergenceTime is None:
minV = min([r.ofValue for r in self.runners])
else:
minV = self.convergenceValue
# if this is not the first method, also should use max from the previous
maxV = self.getBestOfValue()
# return False if exited without a result
if math.isinf(minV) or math.isinf(maxV):
return False
# returns true if either the absolute difference OR relative difference are small
if floatEqual(minV, maxV, epsilonAbs):
return True
# XXX: avoid division by zero; this means relative convergence will always fail on 0.0
if maxV == 0.0:
return False
return abs(1.0 - minV / maxV) < epsilonRel
def start(configFileName):
if not g.prepare(configFileName):
return
# read COPASI model file etc.
strategyManager = strategy.StrategyManager()
if not strategyManager.prepare(isDummy=False):
return
# start the web server
if bool(g.getConfig("web.enable")):
process.createBackgroundThread(executeWebserver, strategyManager)
# start the selected parameter sweep strategy
strategyManager.execute()
def executeWebserver(strategyManager):
try:
port = int(g.getConfig("web.port"))
g.log(LOG_INFO, "<spacescanner>: starting webserver, port: " + str(port))
server = webserver.InterruptibleHTTPServer(('', port), webserver.HttpServerHandler)
server.strategyManager = strategyManager
# report ok and enter the main loop
g.log(LOG_DEBUG, "<spacescanner>: webserver started, listening to port {}".format(port))
server.serve_forever()
except Exception as e:
g.log(LOG_ERROR, "<spacescanner>: exception occurred in webserver:")
g.log(LOG_ERROR, str(e))
g.log(LOG_ERROR, traceback.format_exc())
sys.exit(1)
sys.exit(0)
def start(configFileName):
if not g.prepare(configFileName):
return
# read COPASI model file etc.
strategyManager = strategy.StrategyManager()
if not strategyManager.prepare(isDummy = False):
return
# start the web server
if bool(g.getConfig("web.enable")):
process.createBackgroundThread(executeWebserver, strategyManager)
# start the selected parameter sweep strategy
strategyManager.execute()
def serializeOptimizationTask(self, reportFilename, outf, parameters, methodNames, startParamValues, areParametersChangeable):
# Note: 'scheduled' is always set to true, as is 'update model' to save the final parameter values in the .cps file.
outf.write(' <Task key="{}" name="Optimization" type="optimization" scheduled="true" updateModel="true">\n'.format(self.optimizationTask.get("key")))
# 1. Fix report target file name
# <Report reference="Report_10" target="./report.log" append="1"/>
report = self.optimizationTask.find('copasi:Report', COPASI_NS)
if report is not None:
report.set("target", reportFilename)
report.set("reference", "optimization_report")
outf.write(' ' + str(ElementTree.tostring(report)))
# 2. Include only required parameters
problem = self.optimizationTask.find('copasi:Problem', COPASI_NS)
outf.write(' <Problem>\n')
for elem in problem.iterfind("*", COPASI_NS):
if "Problem" in elem.tag: continue
if "ParameterGroup" not in elem.tag or elem.get("name").lower() != "optimizationitemlist":
if "Parameter" in elem.tag and elem.get("name").lower() == "randomize start values":
# never randomize them
outf.write(' <Parameter name="Randomize Start Values" type="bool" value="0"/>\n')
else:
outf.write(' ' + str(ElementTree.tostring(elem)))
continue
#print("parameters in the file:", self.paramDict.keys())
outf.write(' <ParameterGroup name="OptimizationItemList">\n')
for p in parameters:
if p in self.paramDict:
self.writeParam(outf, p, startParamValues, areParametersChangeable)
outf.write(' </ParameterGroup>\n')
outf.write(' </Problem>\n')
# 3. Include only required methods
#print("methods in the file:", self.methodDict.keys())
for m in methodNames:
methodFromFile = self.methodDict.get(m.lower())
if bool(g.getConfig("methodParametersFromFile")) and methodFromFile is not None:
outf.write(' ' + str(ElementTree.tostring(methodFromFile)))
else:
predefinedMethod = PREDEFINED_METHODS.get(m)
if predefinedMethod is None:
g.log(LOG_ERROR, "Unknown or unsupported optimization method {}".format(m))
else:
outf.write(' ' + predefinedMethod)
# finish off
outf.write('\n </Task>\n')
def ioGetActiveJobs(self, qs):
response = {
"baseline": jsonFixInfinity(self.topBaseline, 0.0),
"stats": []
}
with self.jobLock:
if bool(g.getConfig("webTestMode")):
for id in self.finishedJobs:
if id < 4:
response["stats"].append(
self.finishedJobs[id].getStatsFull())
if self.activeJobPool:
with self.activeJobPool.jobLock:
for job in self.activeJobPool.activeJobs:
if job.areParametersChangeable:
response["stats"].append(job.getStatsFull())
return response
def executeWebserver(strategyManager):
try:
port = int(g.getConfig("web.port"))
g.log(LOG_INFO,
"<spacescanner>: starting webserver, port: " + str(port))
server = webserver.InterruptibleHTTPServer(('', port),
webserver.HttpServerHandler)
server.strategyManager = strategyManager
# report ok and enter the main loop
g.log(
LOG_DEBUG,
"<spacescanner>: webserver started, listening to port {}".format(
port))
server.serve_forever()
except Exception as e:
g.log(LOG_ERROR, "<spacescanner>: exception occurred in webserver:")
g.log(LOG_ERROR, str(e))
g.log(LOG_ERROR, traceback.format_exc())
sys.exit(1)
sys.exit(0)
def main():
if len(sys.argv) > 1 and sys.argv[1] == "web":
# web-only mode; load the last saved web config, if present
g.loadConfig(os.path.join(SELF_PATH, "tmpweb", "config.json"), isQuiet = True)
# create an empty strategy and wait for input commands
strategyManager = strategy.StrategyManager()
strategyManager.prepare(isDummy = True)
# start the web server
process.createBackgroundThread(executeWebserver, strategyManager)
wait()
else:
# normal mode
if not start(sys.argv[1] if len(sys.argv) > 1 else None):
return -1
# if "hang" mode is configured, do not quit until Ctrl+C is pressed
if bool(g.getConfig("hangMode")):
wait()
return 0
def main():
if len(sys.argv) > 1 and sys.argv[1] == "web":
# web-only mode; load the last saved web config, if present
g.loadConfig(os.path.join(SELF_PATH, "tmpweb", "config.json"),
isQuiet=True)
# create an empty strategy and wait for input commands
strategyManager = strategy.StrategyManager()
strategyManager.prepare(isDummy=True)
# start the web server
process.createBackgroundThread(executeWebserver, strategyManager)
wait()
else:
# normal mode
if not start(sys.argv[1] if len(sys.argv) > 1 else None):
return -1
# if "hang" mode is configured, do not quit until Ctrl+C is pressed
if bool(g.getConfig("hangMode")):
wait()
return 0
def execute(self):
parameterSelections = []
if g.getConfig("restartOnFile"):
# Guess which parameters have not been optimized yet based on the .csv result file
filename = g.getConfig("restartOnFile").replace("@SELF@", SELF_PATH)
parameterSets = getNonconvergedResults(filename)
for ps in parameterSets:
spec = {"type" : "explicit", "parameters" : ps}
x = ParamSelection.create(spec, self)
if x not in parameterSelections:
parameterSelections.append(x)
elif g.getConfig("parameters"):
# Deal with TOP, if required
if self.isTOPEnabled():
# add the "zero" at the start
g.log(LOG_INFO, "optimizing for zero parameters initially to find the baseline")
spec = {"type" : "zero"}
parameterSelections.append(ParamSelection.create(spec, self))
# The take the other sets from the file
for spec in g.getConfig("parameters"):
x = ParamSelection.create(spec, self)
if x is None:
g.log(LOG_ERROR, "invalid parameter specification: {}".format(ENC.encode(spec)))
continue
if x not in parameterSelections:
parameterSelections.append(x)
else:
# add the default optimization target: all parameters
g.log(LOG_INFO, "optimizing only for all parameters")
spec = {"type" : "full-set"}
parameterSelections.append(ParamSelection.create(spec, self))
numCombinations = 0
for sel in parameterSelections:
numCombinations += sel.getNumCombinations()
g.log(LOG_INFO, "total {} parameter combination(s) to try out, parameters: {}".format(numCombinations, " ".join(self.copasiConfig["params"])))
g.log(LOG_INFO, "methods enabled: '" + "' '".join(g.getConfig("copasi.methods")) + "'")
parameterSelections.sort(key = lambda x: x.getSortOrder())
for sel in parameterSelections:
areParametersChangeable = sel.type != PARAM_SEL_ZERO
for params in sel.getParameterSets():
g.log(LOG_DEBUG, "made a new pool of {} jobs".format(len(params)))
pool = jobpool.JobPool(self, params, areParametersChangeable)
with self.jobLock:
self.activeJobPool = pool
pool.start()
while True:
time.sleep(1.0)
if self.doQuitFlag:
return True
try:
pool.refresh()
except Exception as e:
g.log(LOG_INFO, "Exception while refreshing active joob pool status, terminating the pool: {}".format(e))
self.finishActivePool()
break
if pool.isDepleted():
self.finishActivePool()
break
return True
def checkReport(self, hasTerminated, now):
assert self.isActive
if not hasTerminated:
if self.lastReportCheckTime is not None and now - self.lastReportCheckTime < MIN_REPORT_CHECK_INTERVAL:
# too soon, skip
return True
if now - self.startTime < 3.0:
# too soon after start (copasi may not be launched yet), return
return True
self.lastReportCheckTime = now
oldOfValue = self.ofValue
try:
st = os.stat(self.reportFilename)
if self.lastReportModificationTime is None or st.st_mtime > self.lastReportModificationTime:
self.lastReportModificationTime = st.st_mtime
with open(self.reportFilename, "r") as f:
self.stats.isValid = False
inValues = False
for line in f:
if startsWith(line, "CPU time"):
inValues = True
continue
if not inValues:
continue
if startsWith(line, "Optimization Result"):
break
si = StatsItem(line)
if si.isValid:
self.stats = si
if self.stats.isValid:
self.ofValue = self.getLastStats().ofValue
if oldOfValue != self.ofValue:
g.log(LOG_INFO, "{}: new OF value {}".format(self.getName(), self.ofValue))
# Check for CPU time end condition using the report file
self.currentCpuTime = self.getLastStats().cpuTime
else:
# no report file update; check for CPU time end condition using OS measurements
if not hasTerminated:
try:
self.currentCpuTime = self.process.getCpuTime()
except psutil.NoSuchProcess:
pass # has already quit
except OSError as e:
g.log(LOG_ERROR, "accessing report file {} failed: {}".format(self.reportFilename, os.strerror(e.errno)))
except IOError as e:
g.log(LOG_ERROR, "parsing report file {} failed: {}".format(self.reportFilename, os.strerror(e.errno)))
if not hasTerminated and not self.terminationReason:
g.log(LOG_DEBUG, "checked {}, CPU time: {}".format(self.getName(), self.currentCpuTime))
# XXX: hardcoded "slow" method names
if self.methodName not in ["ScatterSearch", "SimulatedAnnealing"]:
# XXX: reuse consensus time for this
maxTimeWithNoValue = float(g.getConfig("optimization.consensusMinDurationSec")) + 10.0
if self.job.timeDiffExceeded(now - self.startTime, maxTimeWithNoValue) and (
not self.stats.isValid or self.ofValue == MIN_OF_VALUE
):
g.log(
LOG_DEBUG,
"terminating {}: no value found in CPU time: {}".format(self.getName(), self.currentCpuTime),
)
self.terminationReason = TERMINATION_REASON_CPU_TIME_LIMIT
if oldOfValue != self.ofValue:
# the OF value was updated
return True
return False # the OF value was not updated
def execute(self):
parameterSelections = []
# always add the zero'th job at start, needed to show baseline in graphs and for TOP
g.log(LOG_INFO,
"optimizing for zero parameters initially to find the baseline")
spec = {"type": "zero"}
parameterSelections.append(ParamSelection.create(spec, self))
if self.isTOPEnabled():
# add all parameters: will define the target value
spec = {"type": "full-set"}
parameterSelections.append(ParamSelection.create(spec, self))
if g.getConfig("restartOnFile"):
# Guess which parameters have not been optimized yet based on the .csv result file
filename = g.getConfig("restartOnFile").replace(
"@SELF@", SELF_PATH)
parameterSets = getNonconvergedResults(filename)
for ps in parameterSets:
spec = {"type": "explicit", "parameters": ps}
x = ParamSelection.create(spec, self)
if x not in parameterSelections:
parameterSelections.append(x)
elif g.getConfig("parameters"):
# Take the paramter sets from the file
for spec in g.getConfig("parameters"):
x = ParamSelection.create(spec, self)
if x is None:
g.log(
LOG_ERROR,
"invalid parameter specification: {}".format(
ENC.encode(spec)))
continue
if x not in parameterSelections:
parameterSelections.append(x)
else:
# add the default optimization target: all parameters
g.log(LOG_INFO, "optimizing only for all parameters")
spec = {"type": "full-set"}
parameterSelections.append(ParamSelection.create(spec, self))
hashes = set()
for sel in parameterSelections:
hashes = hashes.union(sel.getAllJobHashes())
self.totalNumJobs = len(hashes) - 1
g.log(
LOG_INFO,
"total {} parameter combination(s) to try out, parameters: {}".
format(self.totalNumJobs, " ".join(self.copasiConfig["params"])))
g.log(
LOG_INFO, "methods enabled: '" +
"' '".join(g.getConfig("copasi.methods")) + "'")
parameterSelections.sort(key=lambda x: x.getSortOrder())
for sel in parameterSelections:
g.log(LOG_DEBUG,
"processing parameter selection of type {}".format(sel.type))
for params in sel.getParameterSets():
g.log(LOG_DEBUG,
"made a new pool of {} jobs".format(len(params)))
pool = jobpool.JobPool(self, params,
sel.areParametersChangeable())
with self.jobLock:
self.activeJobPool = pool
pool.start()
while True:
time.sleep(1.0)
if self.doQuitFlag:
return True
try:
pool.refresh()
except Exception as e:
g.log(
LOG_INFO,
"Exception while refreshing active joob pool status, terminating the pool: {}"
.format(e))
self.finishActivePool()
break
if pool.isDepleted():
self.finishActivePool()
break
return True
def checkReports(self):
# if no runners are active, quit
if not any([r.isActive for r in self.runners]):
if self.convergenceTime is not None:
minAbsoluteTime = float(g.getConfig("optimization.consensusMinDurationSec"))
# XXX: do not check the relative time here
if self.hasConsensus() and self.timeDiffExceeded(time.time() - self.convergenceTime, minAbsoluteTime):
# count COPASI termination as consensus in this case
# XXX: note that this does *not* overwrite "time limit exceeded" exit code!
for r in self.runners:
if r.terminationReason == TERMINATION_REASON_COPASI_FINISHED:
r.terminationReason = TERMINATION_REASON_CONSENSUS
# switch the methods if required
self.decideTermination()
return
if not all([r.isActive for r in self.runners]):
# some but not all have quit; quit the others with "stagnation"
# (not technically true, but the best match from the existing codes)
for r in self.runners:
if r.isActive and not r.terminationReason:
r.terminationReason = TERMINATION_REASON_STAGNATION
return
numActiveRunners = 0
now = time.time()
cpuTimeLimit = float(g.getConfig("optimization.timeLimitSec"))
maxCpuTime = 0
anyUpdated = False
with runner.reportLock:
for r in self.runners:
if r.isActive:
numActiveRunners += 1
if r.checkReport(hasTerminated=False, now=now):
# the runner updated the OF time
self.lastOfUpdateTime = now
anyUpdated = True
maxCpuTime = max(maxCpuTime, r.currentCpuTime)
if not self.areParametersChangeable:
# it is simple; as soon as the first value is read, return
if anyUpdated:
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CONSENSUS
return
consensusReached = self.hasConsensus()
if all([r.terminationReason for r in self.runners]):
return
# use the old CPU time as a basis
maxCpuTime += sum(self.oldCpuTimes)
doKillOnTimeLimit = maxCpuTime >= cpuTimeLimit and not consensusReached
if doKillOnTimeLimit:
# kill all jobs immediately
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CPU_TIME_LIMIT
g.log(
LOG_INFO,
"terminating {}: CPU time limit exceeded ({} vs. {})".format(
r.getName(), r.currentCpuTime, cpuTimeLimit
),
)
return
# check if the runs have reached consensus
if consensusReached:
if self.convergenceTime is None:
g.log(LOG_DEBUG, self.getName() + ": reached consensus, waiting for guard time before termination")
self.convergenceTime = now
self.convergenceValue = min([r.ofValue for r in self.runners])
# if the runners have converged for long enough time, quit
else:
timeConverged = time.time() - self.convergenceTime
minAbsoluteTime = float(g.getConfig("optimization.consensusMinDurationSec"))
minRelativeTime = (time.time() - self.startTime) * float(
g.getConfig("optimization.consensusMinProportionalDuration")
)
if self.timeDiffExceeded(timeConverged, minAbsoluteTime) and timeConverged > minRelativeTime:
g.log(LOG_INFO, "terminating {}: consensus reached".format(self.getName()))
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CONSENSUS
self.convergenceTime = now
return # do not check other criteria
else:
# reset the timer
self.convergenceTime = None
# check for stagnation's time limit
timeStagnated = time.time() - self.lastOfUpdateTime
maxAbsoluteTime = float(g.getConfig("optimization.stagnationMaxDurationSec"))
maxRelativeTime = (time.time() - self.startTime) * float(
g.getConfig("optimization.stagnationMaxProportionalDuration")
)
if self.timeDiffExceeded(timeStagnated, maxAbsoluteTime) and timeStagnated > maxRelativeTime:
# stagnation detected
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_STAGNATION
g.log(
LOG_INFO,
"terminating {}: Optimization stagnated (did not produce new results) for {} seconds".format(
self.getName(), timeStagnated
),
)
return
# We will continue. Check if load balancing needs to be done
if now - self.lastBalanceTime >= LOAD_BALANCE_INTERVAL:
self.lastBalanceTime = now
if numActiveRunners > self.maxCores:
# not converged yet + too many active; limit some runners
cpuTimes = [(r.currentCpuTime, r) for r in self.runners if r.isActive]
cpuTimes.sort()
# continue first `maxCores` runners, suspend the rest
resumeRunners = cpuTimes[: self.maxCores]
suspendRunners = cpuTimes[self.maxCores :]
for _, j in resumeRunners:
j.suspend(False)
for _, j in suspendRunners:
j.suspend(True)
else:
for r in self.runners:
r.suspend(False)
def decideTermination(self):
continuableReasons = [TERMINATION_REASON_STAGNATION, TERMINATION_REASON_COPASI_FINISHED]
if not any([r.terminationReason in continuableReasons for r in self.runners]):
# all terminated with consensus, because asked by the user, or with time limit
self.pool.finishJob(self)
return
# So we have at least one termination with either:
# a) the stagnation limit was exceeded, or
# b) Copasi stopped without consensus.
# Actions now:
# 1) if no solution found: use a fallback method;
# 2) else switch to the next method;
# 3) if no more methods are available, quit.
if self.currentMethod in self.fallbackMethods:
# remove the already-used method to avoid infinite looping between methods
self.fallbackMethods.remove(self.currentMethod)
assert self.currentMethod in self.methods
self.methods.remove(self.currentMethod)
anyNotFound = any([math.isinf(r.ofValue) for r in self.runners])
if anyNotFound or self.isUsingFallback:
if len(self.fallbackMethods) == 0:
if anyNotFound:
g.log(LOG_INFO, "terminating {}: failed to evaluate the objective function".format(self.getName()))
else:
g.log(
LOG_INFO,
"terminating {}: all fallback methods exhausted without reaching consensus".format(
self.getName()
),
)
self.pool.finishJob(self)
return
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.fallbackMethods)
else:
self.currentMethod = self.fallbackMethods[0]
# make sure the fallback methods are also in methods
if self.currentMethod not in self.methods:
self.methods.append(self.currentMethod)
g.log(LOG_INFO, "switching {} to a fallback method {}".format(self.getName(), self.currentMethod))
self.isUsingFallback = True
# switch to the fallback method
if not self.createRunners():
self.pool.finishJob(self)
return
if len(self.methods) == 0:
g.log(LOG_INFO, "terminating {}: all methods exhausted without reaching consensus".format(self.getName()))
self.pool.finishJob(self)
return
# go for the next method
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.methods)
else:
self.currentMethod = self.methods[0]
g.log(LOG_INFO, "switching {} to the next method {}".format(self.getName(), self.currentMethod))
if not self.createRunners():
self.pool.finishJob(self)
return
def serializeOptimizationTask(self, reportFilename, outf, parameters,
methodNames, startParamValues,
areParametersChangeable):
# Note: 'scheduled' is always set to true, as is 'update model' to save the final parameter values in the .cps file.
outf.write(
' <Task key="{}" name="Optimization" type="optimization" scheduled="true" updateModel="true">\n'
.format(self.optimizationTask.get("key")))
# 1. Fix report target file name
# <Report reference="Report_10" target="./report.log" append="1"/>
report = self.optimizationTask.find('copasi:Report', COPASI_NS)
if report is not None:
report.set("target", reportFilename)
report.set("reference", "optimization_report")
outf.write(' ' + str(ElementTree.tostring(report)))
# 2. Include only required parameters
problem = self.optimizationTask.find('copasi:Problem', COPASI_NS)
outf.write(' <Problem>\n')
for elem in problem.iterfind("*", COPASI_NS):
if "Problem" in elem.tag: continue
if "ParameterGroup" not in elem.tag or elem.get(
"name").lower() != "optimizationitemlist":
if "Parameter" in elem.tag and elem.get(
"name").lower() == "randomize start values":
# never randomize them
outf.write(
' <Parameter name="Randomize Start Values" type="bool" value="0"/>\n'
)
else:
outf.write(' ' + str(ElementTree.tostring(elem)))
continue
#print("parameters in the file:", self.paramDict.keys())
outf.write(' <ParameterGroup name="OptimizationItemList">\n')
for p in parameters:
if p in self.paramDict:
self.writeParam(outf, p, startParamValues,
areParametersChangeable)
outf.write(' </ParameterGroup>\n')
outf.write(' </Problem>\n')
# 3. Include only required methods
#print("methods in the file:", self.methodDict.keys())
for m in methodNames:
methodFromFile = self.methodDict.get(m.lower())
if bool(g.getConfig("methodParametersFromFile")
) and methodFromFile is not None:
outf.write(' ' +
str(ElementTree.tostring(methodFromFile)))
else:
predefinedMethod = PREDEFINED_METHODS.get(m)
if predefinedMethod is None:
g.log(
LOG_ERROR,
"Unknown or unsupported optimization method {}".format(
m))
else:
outf.write(' ' + predefinedMethod)
# finish off
outf.write('\n </Task>\n')
def checkReports(self):
# if no runners are active, quit
if not any([r.isActive for r in self.runners]):
if self.convergenceTime is not None:
minAbsoluteTime = float(g.getConfig("optimization.consensusDelaySec"))
# XXX: do not check the relative time here
if self.hasConsensus() and self.timeDiffExceeded(time.time() - self.convergenceTime, minAbsoluteTime):
# count COPASI termination as consensus in this case
# XXX: note that this does *not* overwrite "time limit exceeded" exit code!
for r in self.runners:
if r.terminationReason == TERMINATION_REASON_COPASI_FINISHED:
r.terminationReason = TERMINATION_REASON_CONSENSUS
# switch the methods if required
self.decideTermination()
return
if not all([r.isActive for r in self.runners]):
# some but not all have quit; quit the others with "stagnation"
# (not technically true, but the best match from the existing codes)
for r in self.runners:
if r.isActive and not r.terminationReason:
r.terminationReason = TERMINATION_REASON_STAGNATION
return
numActiveRunners = 0
now = time.time()
cpuTimeLimit = float(g.getConfig("optimization.timeLimitSec"))
maxCpuTime = 0
anyUpdated = False
with runner.reportLock:
for r in self.runners:
if r.isActive:
numActiveRunners += 1
if r.checkReport(hasTerminated = False, now = now):
# the runner updated the OF time
self.lastOfUpdateTime = now
anyUpdated = True
maxCpuTime = max(maxCpuTime, r.currentCpuTime)
if not self.areParametersChangeable:
# it is simple; as soon as the first value is read, return
if anyUpdated:
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CONSENSUS
return
consensusReached = self.hasConsensus()
if all([r.terminationReason for r in self.runners]):
return
# use the old CPU time as a basis
maxCpuTime += sum(self.oldCpuTimes)
doKillOnTimeLimit = maxCpuTime >= cpuTimeLimit and not consensusReached
if doKillOnTimeLimit:
# kill all jobs immediately
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CPU_TIME_LIMIT
g.log(LOG_INFO, "terminating {}: CPU time limit exceeded ({} vs. {})".format(
r.getName(), maxCpuTime, cpuTimeLimit))
return
# check if the runs have reached consensus
if consensusReached:
if self.convergenceTime is None:
g.log(LOG_DEBUG, self.getName() + ": reached consensus, waiting for guard time before termination")
self.convergenceTime = now
self.convergenceValue = min([r.ofValue for r in self.runners])
# if the runners have converged for long enough time, quit
else:
timeConverged = time.time() - self.convergenceTime
minAbsoluteTime = float(g.getConfig("optimization.consensusDelaySec"))
minRelativeTime = (time.time() - self.startTime) * float(g.getConfig("optimization.consensusProportionalDelay"))
if self.timeDiffExceeded(timeConverged, minAbsoluteTime) and timeConverged > minRelativeTime:
g.log(LOG_INFO, "terminating {}: consensus reached".format(self.getName()))
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CONSENSUS
self.convergenceTime = now
return # do not check other criteria
else:
# reset the timer
self.convergenceTime = None
# check for stagnation's time limit
timeStagnated = time.time() - self.lastOfUpdateTime
maxAbsoluteTime = float(g.getConfig("optimization.stagnationDelaySec"))
maxRelativeTime = (time.time() - self.startTime) * float(g.getConfig("optimization.stagnationProportionalDelay"))
# XXX: specialcase for the non-parameters job: quit it quite quickly (in 10 seconds for each method)
if not self.areParametersChangeable:
maxAbsoluteTime = 10.0
maxRelativeTime = 0.0
if self.timeDiffExceeded(timeStagnated, maxAbsoluteTime) and timeStagnated > maxRelativeTime:
# stagnation detected
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_STAGNATION
g.log(LOG_INFO, "terminating {}: Optimization stagnated (did not produce new results) for {} seconds".format(self.getName(), timeStagnated))
return
# We will continue. Check if load balancing needs to be done
if now - self.lastBalanceTime >= LOAD_BALANCE_INTERVAL:
self.lastBalanceTime = now
if numActiveRunners > self.maxCores:
# not converged yet + too many active; limit some runners
cpuTimes = [(r.currentCpuTime, r) for r in self.runners if r.isActive]
cpuTimes.sort()
# continue first `maxCores` runners, suspend the rest
resumeRunners = cpuTimes[:self.maxCores]
suspendRunners = cpuTimes[self.maxCores:]
for _,j in resumeRunners:
j.suspend(False)
for _,j in suspendRunners:
j.suspend(True)
else:
for r in self.runners:
r.suspend(False)
def decideTermination(self):
continuableReasons = [TERMINATION_REASON_STAGNATION,
TERMINATION_REASON_COPASI_FINISHED]
if not any([r.terminationReason in continuableReasons for r in self.runners]):
# all terminated with consensus, because asked by the user, or with time limit
self.pool.finishJob(self)
return
# So we have at least one termination with either:
# a) the stagnation limit was exceeded, or
# b) Copasi stopped without consensus.
# Actions now:
# 1) if no solution found: use a fallback method;
# 2) else switch to the next method;
# 3) if no more methods are available, quit.
if self.currentMethod in self.fallbackMethods:
# remove the already-used method to avoid infinite looping between methods
self.fallbackMethods.remove(self.currentMethod)
assert (self.currentMethod in self.methods)
self.methods.remove(self.currentMethod)
anyNotFound = any([math.isinf(r.ofValue) for r in self.runners])
if anyNotFound or self.isUsingFallback:
if len(self.fallbackMethods) == 0:
if anyNotFound:
g.log(LOG_INFO, "terminating {}: failed to evaluate the objective function".format(self.getName()))
else:
g.log(LOG_INFO, "terminating {}: all fallback methods exhausted without reaching consensus".format(self.getName()))
self.pool.finishJob(self)
return
self.pastMethods.append(self.currentMethod)
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.fallbackMethods)
else:
self.currentMethod = self.fallbackMethods[0]
# make sure the fallback methods are also in methods
if self.currentMethod not in self.methods:
self.methods.append(self.currentMethod)
g.log(LOG_INFO, "switching {} to a fallback method {}".format(self.getName(), self.currentMethod))
self.isUsingFallback = True
# switch to the fallback method
if not self.createRunners():
self.pool.finishJob(self)
return
if len(self.methods) == 0:
g.log(LOG_INFO, "terminating {}: all methods exhausted without reaching consensus".format(self.getName()))
self.pool.finishJob(self)
return
# go for the next method
self.pastMethods.append(self.currentMethod)
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.methods)
else:
self.currentMethod = self.methods[0]
g.log(LOG_INFO, "switching {} to the next method {}".format(
self.getName(), self.currentMethod))
if not self.createRunners():
self.pool.finishJob(self)
return
