def nextJob(self):
if self.currentParametersIndex >= len(self.parameterSets):
return
params = self.parameterSets[self.currentParametersIndex]
self.currentParametersIndex += 1
# check if a job with parameters has already been around
hash = getParamSetHash(params, self.strategy.copasiConfig["params"], not self.areParametersChangeable)
if hash in self.strategy.startedJobs:
g.log(LOG_DEBUG, "skipping a job, parameter set already processed: {}".format(params))
return
self.strategy.startedJobs.add(hash)
numFreeCores = self.numUsableCores
with self.jobLock:
for j in self.activeJobs:
numFreeCores -= j.maxCores
numFreeCores = max(1, numFreeCores)
# setup a new job
j = job.Job(self,
params, # the set of parameters
min(numFreeCores, self.numRunnersPerJob), # the number of simultaneous processes
self.areParametersChangeable) # whether to enable optimization
# add it to the list of active jobs
with self.jobLock:
self.activeJobs.append(j)
# execute the job
if not j.execute(g.workDir, self.strategy.copasiFile):
g.log(LOG_DEBUG, "failed to execute {}".format(j.getName()))
self.finishJob(j)
def prepare(self, isDummy):
self.jobLock = threading.Lock()
self.activeJobPool = None
self.finishedJobs = {}
self.startedJobs = set()
self.doQuitFlag = False
self.isExecutable = False
self.lastNumJobsDumped = 0
# job counter, starting from 1
self.nextJobID = itertools.count(1)
self.copasiConfig = {"params" : []}
self.jobsByBestOfValue = []
self.topBaseline = 0.0
self.copasiFile = self.loadCopasiFile()
if not self.copasiFile:
return False
g.log(LOG_DEBUG, "querying COPASI optimization parameters")
self.copasiConfig["params"] = self.copasiFile.queryParameters()
if not self.copasiConfig["params"]:
return False
self.jobsByBestOfValue = [[] for _ in range(1 + len(self.copasiConfig["params"]))]
if not isDummy:
self.isExecutable = True
return True
def executeCopasi(runner):
# a pseudo-loop for simpler error handling
while True:
if bool(g.getConfig("webTestMode")):
if runner.job.id < 4:
time.sleep(1.0)
runner.ofValue = random.random() * 100
runner.isActive = False
else:
# wait for the strategy to quit
while not runner.job.pool.strategy.doQuitFlag:
time.sleep(1.0)
runner.ofValue += random.random()
runner.isActive = False
break
g.log(LOG_DEBUG, "executing " + " ".join(runner.process.args))
runner.isError = runner.process.run()
# check report in order to update OF value (even if nonzero return value)
with reportLock:
runner.checkReport(hasTerminated=True, now=time.time())
# exit the loop without an error
break
# check termination conditions at the global run level
g.log(LOG_DEBUG, "{}: terminated".format(runner.getName()))
runner.isActive = False
# disable this; do it from polling in the main thread instead
#runner.job.checkIfHasTerminated()
# overwrite the .cps file with best parameter values
runner.cleanup()
def createRunners(self):
# move the current runners, if any, to the array with old runners
if self.runners:
self.oldCpuTimes.append(max(r.currentCpuTime for r in self.runners))
self.oldRunners.extend(self.runners)
self.runners = []
# reset other state
self.convergenceTime = None
self.lastOfUpdateTime = time.time()
self.runnerGeneration += 1
bestParams = None
if self.oldRunners and bool(g.getConfig("optimization.restartFromBestValue")):
# get best params
bestParams = self.getBestParams()
for id in range(int(g.getConfig("optimization.runsPerJob"))):
r = runner.Runner(self, id + 1, self.currentMethod, self.runnerGeneration)
if not r.prepare(self.workDir, self.copasiFile, bestParams):
g.log(LOG_ERROR, "{}: failed to create a runner".format(r.getName()))
return False
self.runners.append(r)
# note that this may create more processes than the number of free CPU cores!
for r in self.runners:
r.execute()
return True
def suspend(self, yes):
if yes:
g.log(LOG_DEBUG, "suspending process " + " ".join(self.args))
self.psutilProcess.suspend()
else:
g.log(LOG_DEBUG, "resuming process " + " ".join(self.args))
self.psutilProcess.resume()
def getAllStatsForGeneration(self, generation):
result = []
try:
filename = self.reportFilename
if generation != self.generation:
filename += "_gen" + str(generation)
with open(filename, "r") as f:
inValues = False
for line in f:
if startsWith(line, "CPU time"):
inValues = True
continue
if not inValues: continue
if startsWith(line, "Optimization Result"):
break
si = StatsItem(line)
if si.isValid:
result.append(si)
except IOError as e:
g.log(LOG_DEBUG, "failed to read a report file " + filename)
return result
def getAllStatsForGeneration(self, generation):
result = []
try:
filename = self.reportFilename
if generation != self.generation:
filename += "_gen" + str(generation)
with open(filename, "r") as f:
inValues = False
for line in f:
if startsWith(line, "CPU time"):
inValues = True
continue
if not inValues:
continue
if startsWith(line, "Optimization Result"):
break
si = StatsItem(line)
if si.isValid:
result.append(si)
except IOError as e:
g.log(LOG_DEBUG, "failed to read a report file " + filename)
return result
def executeCopasi(runner):
# a pseudo-loop for simpler error handling
while True:
if bool(g.getConfig("webTestMode")):
if runner.job.id < 4:
time.sleep(1.0)
runner.ofValue = random.random() * 100
runner.isActive = False
else:
# wait for the strategy to quit
while not runner.job.pool.strategy.doQuitFlag:
time.sleep(1.0)
runner.ofValue += random.random()
runner.isActive = False
break
g.log(LOG_DEBUG, "executing " + " ".join(runner.process.args))
runner.isError = runner.process.run()
# check report in order to update OF value (even if nonzero return value)
with reportLock:
runner.checkReport(hasTerminated=True, now=time.time())
# exit the loop without an error
break
# check termination conditions at the global run level
g.log(LOG_DEBUG, "{}: terminated".format(runner.getName()))
runner.isActive = False
# disable this; do it from polling in the main thread instead
# runner.job.checkIfHasTerminated()
# overwrite the .cps file with best parameter values
runner.cleanup()
def createRunners(self):
# move the current runners, if any, to the array with old runners
if self.runners:
self.oldCpuTimes.append(max(r.currentCpuTime for r in self.runners))
self.oldRunners.extend(self.runners)
self.runners = []
# reset other state
self.convergenceTime = None
self.lastOfUpdateTime = time.time()
self.runnerGeneration += 1
bestParams = None
if self.oldRunners and bool(g.getConfig("optimization.restartFromBestValue")):
# get best params
bestParams = self.getBestParams()
for id in range(int(g.getConfig("optimization.runsPerJob"))):
r = runner.Runner(self, id + 1, self.currentMethod, self.runnerGeneration)
if not r.prepare(self.workDir, self.copasiFile, bestParams):
g.log(LOG_ERROR, "{}: failed to create a runner".format(r.getName()))
return False
self.runners.append(r)
# note that this may create more processes than the number of free CPU cores!
for r in self.runners:
r.execute()
return True
def nextJob(self):
if self.currentParametersIndex >= len(self.parameterSets):
return
params = self.parameterSets[self.currentParametersIndex]
self.currentParametersIndex += 1
# check if a job with parameters has already been around
hash = getParamSetHash(params, self.strategy.copasiConfig["params"])
if hash in self.strategy.startedJobs:
g.log(LOG_DEBUG, "skipping a job, parameter set already processed: {}".format(params))
return
self.strategy.startedJobs.add(hash)
numFreeCores = self.numUsableCores
with self.jobLock:
for j in self.activeJobs:
numFreeCores -= j.maxCores
numFreeCores = max(1, numFreeCores)
# setup a new job
j = job.Job(self,
params, # the set of parameters
min(numFreeCores, self.numRunnersPerJob), # the number of simultaneous processes
self.areParametersChangeable) # whether to enable optimization
# add it to the list of active jobs
with self.jobLock:
self.activeJobs.append(j)
# execute the job
if not j.execute(g.workDir, self.strategy.copasiFile):
g.log(LOG_DEBUG, "failed to execute {}".format(j.getName()))
self.finishJob(j)
def prepare(self, isDummy):
self.jobLock = threading.Lock()
self.activeJobPool = None
self.finishedJobs = {}
self.startedJobs = set()
self.doQuitFlag = False
self.isExecutable = False
self.lastNumJobsDumped = 0
# job counter, starting from 0
self.nextJobID = 0
self.copasiConfig = {"params": []}
self.jobsByBestOfValue = []
self.topBaseline = 0.0
self.copasiFile = self.loadCopasiFile()
if not self.copasiFile:
return False
g.log(LOG_DEBUG, "querying COPASI optimization parameters")
self.copasiConfig["params"] = self.copasiFile.queryParameters()
if not self.copasiConfig["params"]:
return False
self.jobsByBestOfValue = [
[] for _ in range(1 + len(self.copasiConfig["params"]))
]
if not isDummy:
self.isExecutable = True
return True
def loadCopasiFile(self):
copasiFile = copasifile.CopasiFile()
filename = g.getConfig("copasi.modelFile")
filename = filename.replace("@SELF@", SELF_PATH)
g.log(LOG_INFO, "<spacescanner>: opening COPASI model file {}".format(filename))
if not copasiFile.read(filename):
return None
return copasiFile
def loadCopasiFile(self):
copasiFile = copasifile.CopasiFile()
filename = g.getConfig("copasi.modelFile")
filename = filename.replace("@SELF@", SELF_PATH)
g.log(LOG_INFO,
"<spacescanner>: opening COPASI model file {}".format(filename))
if not copasiFile.read(filename):
return None
return copasiFile
def finishJob(self, j):
g.log(LOG_DEBUG, "finished {}".format(j.getName()))
with self.jobLock:
if j not in self.activeJobs:
return
self.activeJobs.remove(j)
if self.bestOfValue < j.getBestOfValue():
# improved on the OF value! Store the result now.
self.bestOfValue = j.getBestOfValue()
self.bestParams = copy.copy(j.params)
self.strategy.finishJob(j)
def dumpResults(self, totalLimit=0, perParamLimit=0):
if g.workDir is None:
return None
filename = g.getConfig("output.filename")
# do not allow put the results in other directories because of security reasons
if filename != os.path.basename(filename):
g.log(
LOG_INFO,
"output file name should not include path, ignoring all but the last element in it"
)
filename = os.path.basename(filename)
(name, ext) = os.path.splitext(filename)
if not ext:
ext = ".csv" # default
filename = os.path.join(g.workDir,
"{}-{}{}".format(name, "-", g.taskName, ext))
with self.jobLock:
if len(self.finishedJobs) <= self.lastNumJobsDumped:
# all finished jobs already were saved, nothing to do
return filename
self.lastNumJobsDumped = len(self.finishedJobs)
allJobsByBestOfValue = []
if perParamLimit == 0:
numberOfBestCombinations = int(
g.getConfig("output.numberOfBestCombinations"))
else:
numberOfBestCombinations = perParamLimit
for joblist in self.jobsByBestOfValue:
if numberOfBestCombinations:
lst = joblist[:numberOfBestCombinations]
else:
lst = joblist
for job in lst:
allJobsByBestOfValue.append(job)
allJobsByBestOfValue.sort(key=lambda x: x.getBestOfValue(),
reverse=True)
allParams = self.copasiConfig["params"]
cnt = 0
with open(filename, "w") as f:
self.dumpCsvFileHeader(f)
for job in allJobsByBestOfValue:
job.dumpResults(f, allParams)
cnt += 1
if totalLimit and cnt >= totalLimit:
break
g.log(
LOG_INFO, '<spacescanner>: results of finished jobs saved in "' +
filename + '"')
return filename
def finishJob(self, j):
g.log(LOG_DEBUG, "finished {}".format(j.getName()))
with self.jobLock:
if j not in self.activeJobs:
return
self.activeJobs.remove(j)
if self.bestOfValue < j.getBestOfValue():
# improved on the OF value! Store the result now.
self.bestOfValue = j.getBestOfValue()
self.bestParams = copy.copy(j.params)
self.strategy.finishJob(j)
def execute(self, workDir, copasiFile):
self.workDir = workDir
self.copasiFile = copasiFile
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.methods)
else:
self.currentMethod = self.methods[0]
g.log(LOG_INFO, "starting " + self.getFullName())
return self.createRunners()
def execute(self, workDir, copasiFile):
self.workDir = workDir
self.copasiFile = copasiFile
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.methods)
else:
self.currentMethod = self.methods[0]
g.log(LOG_INFO, "starting " + self.getFullName())
return self.createRunners()
def getBestParameters(self, k):
joblist = self.jobsByBestOfValue[k]
if not joblist:
if 0:
g.log(
LOG_ERROR,
"Parameter ranges are invalid: best value of {} parameters requested, but no jobs finished"
.format(k))
return None
else:
# this is fine and expected for preliminary calculations
return self.copasiConfig["params"][:k]
return joblist[0].params
def writeParam(self, outf, param, startParamValues,
areParametersChangeable):
xml = self.paramDict[param]
if areParametersChangeable:
if startParamValues is None or param not in startParamValues:
# no need to preprocess; write directly back in the file
outf.write(' ' + str(ElementTree.tostring(xml)))
return
else:
if startParamValues is None:
startParamValues = {}
for sub in xml.iterfind("*", COPASI_NS):
if "ParameterGroup" in sub.tag: continue
if sub.get("name").lower(
) == "startvalue" and param not in startParamValues:
# read it directly from the file
try:
v = sub.get("value")
startParamValues[param] = float(v)
except:
g.log(
LOG_ERROR, "Getting start value of a parameter " +
param + " failed:" + sub.get("value"))
outf.write(' <ParameterGroup name="OptimizationItem">\n')
for sub in xml.iterfind("*", COPASI_NS):
if "ParameterGroup" in sub.tag: continue
if sub.get("name").lower() == "startvalue" \
and startParamValues is not None \
and param in startParamValues:
outf.write(
' <Parameter name="StartValue" type="float" value="{}"/>\n'
.format(startParamValues[param]))
continue
if sub.get("name").lower() in ["lowerbound", "upperbound"] \
and not areParametersChangeable \
and startParamValues is not None \
and param in startParamValues:
# unchangeable; set the bounds to the start value
val = startParamValues[param]
outf.write(
' <Parameter name="{}" type="cn" value="{}"/>\n'.
format(sub.get("name"), val))
continue
# by default, just write back whatever was in the file
outf.write(' ' + str(ElementTree.tostring(sub)))
outf.write(' </ParameterGroup>\n')
def ioGetResults(self, qs):
totalLimit = int(qs.get("totallimit", 0))
perParamLimit = int(qs.get("perparamlimit", 0))
filename = self.dumpResults(totalLimit, perParamLimit)
contents = ""
try:
with open(filename) as f:
contents = f.read()
except IOError as e:
g.log(LOG_DEBUG, "failed to read result .csv file {}".format(filename))
except Exception as e:
g.log(LOG_INFO, "failed to read result .csv file {}: {}".format(filename, e))
return contents
def read(self, filename):
if not isReadable(filename):
g.log(LOG_ERROR, "error while loading COPASI model: file not found or not readable")
return False
self.optimizationTask = None
self.xmlroot = ElementTree.parse(filename).getroot()
# Example XML structure:
# <xml><ListOfTasks><Task type="optimization">...
for tasklist in self.xmlroot.findall('copasi:ListOfTasks', COPASI_NS):
for task in tasklist.findall('copasi:Task', COPASI_NS):
if task.get("type").lower() == "optimization":
self.optimizationTask = task
if self.optimizationTask is None:
g.log(LOG_ERROR, "error while loading COPASI model: optimization task not found in COPASI file")
return False
self.loadParameters()
if len(self.paramDict) == 0:
g.log(LOG_ERROR, "error while loading COPASI model: optimization parameters not defined in COPASI file")
return False
if not self.objectiveFunction:
g.log(LOG_ERROR, "error while loading COPASI model: objective function not defined in COPASI file")
return False
return True
def executeWebserver(strategyManager):
try:
port = int(g.getConfig("web.port"))
g.log(LOG_INFO, "<spacescanner>: starting webserver, port: " + str(port))
server = webserver.InterruptibleHTTPServer(('', port), webserver.HttpServerHandler)
server.strategyManager = strategyManager
# report ok and enter the main loop
g.log(LOG_DEBUG, "<spacescanner>: webserver started, listening to port {}".format(port))
server.serve_forever()
except Exception as e:
g.log(LOG_ERROR, "<spacescanner>: exception occurred in webserver:")
g.log(LOG_ERROR, str(e))
g.log(LOG_ERROR, traceback.format_exc())
sys.exit(1)
sys.exit(0)
def startFromWeb(configFileName):
if not g.prepare(configFileName):
g.log(LOG_ERROR, "Preparing from config file failed: " + configFileName)
return None
# read COPASI model file etc.
strategyManager = strategy.StrategyManager()
if not strategyManager.prepare(isDummy = False):
g.log(LOG_ERROR, "Preparing for execution failed")
return None
# start the selected parameter sweep strategy asynchronously
process.createBackgroundThread(lambda s: s.execute(), strategyManager)
return strategyManager
def loadParameters(self):
assert self.optimizationTask is not None
problem = self.optimizationTask.find('copasi:Problem', COPASI_NS)
if problem is None:
g.log(LOG_ERROR, "'Problem' not found in the optimization task")
return []
self.paramDict = {}
self.methodDict = {}
# Example XML structure:
# Task type="optimization"><ParameterGroup name="OptimizationItem"><Parameter> ...
for paramGroup in problem.findall('copasi:ParameterGroup', COPASI_NS):
if paramGroup.get("name").lower() == "optimizationitemlist":
for paramGroup2 in paramGroup.findall('copasi:ParameterGroup',
COPASI_NS):
if paramGroup2.get("name").lower() != "optimizationitem":
continue
# Example XML syntax:
# <Parameter name="ObjectCN" type="cn" value="CN=Root,Model=Galazzo1990_FermentationPathwayKinetics,Vector=Reactions[Pyruvate kinase],ParameterGroup=Parameters,Parameter=Vm6,Reference=Value"/>
for param in paramGroup2.findall('copasi:Parameter',
COPASI_NS):
if param.get("name").lower() != "objectcn": continue
val = param.get("value")
if not val: continue
val = [x.split("=") for x in val.split(",")]
for (k, v) in val:
if 0:
if k.lower() == "parameter":
self.paramDict["'" + v + "'"] = paramGroup2
else:
if k.lower() == "vector":
if "[" in v and "]" in v:
v = v[v.find("[") + 1:v.find("]")]
self.paramDict["'" + v + "'"] = paramGroup2
self.objectiveFunction = None
for paramText in problem.findall('copasi:ParameterText', COPASI_NS):
if paramText.get("name").lower() == "objectiveexpression":
self.objectiveFunction = paramText.text.strip()
# parse methods as well
for method in self.optimizationTask.findall('copasi:Method',
COPASI_NS):
mtype = method.get("type").lower()
self.methodDict[mtype] = method
def serializeOptimizationTask(self, reportFilename, outf, parameters, methodNames, startParamValues, areParametersChangeable):
# Note: 'scheduled' is always set to true, as is 'update model' to save the final parameter values in the .cps file.
outf.write(' <Task key="{}" name="Optimization" type="optimization" scheduled="true" updateModel="true">\n'.format(self.optimizationTask.get("key")))
# 1. Fix report target file name
# <Report reference="Report_10" target="./report.log" append="1"/>
report = self.optimizationTask.find('copasi:Report', COPASI_NS)
if report is not None:
report.set("target", reportFilename)
report.set("reference", "optimization_report")
outf.write(' ' + str(ElementTree.tostring(report)))
# 2. Include only required parameters
problem = self.optimizationTask.find('copasi:Problem', COPASI_NS)
outf.write(' <Problem>\n')
for elem in problem.iterfind("*", COPASI_NS):
if "Problem" in elem.tag: continue
if "ParameterGroup" not in elem.tag or elem.get("name").lower() != "optimizationitemlist":
if "Parameter" in elem.tag and elem.get("name").lower() == "randomize start values":
# never randomize them
outf.write(' <Parameter name="Randomize Start Values" type="bool" value="0"/>\n')
else:
outf.write(' ' + str(ElementTree.tostring(elem)))
continue
#print("parameters in the file:", self.paramDict.keys())
outf.write(' <ParameterGroup name="OptimizationItemList">\n')
for p in parameters:
if p in self.paramDict:
self.writeParam(outf, p, startParamValues, areParametersChangeable)
outf.write(' </ParameterGroup>\n')
outf.write(' </Problem>\n')
# 3. Include only required methods
#print("methods in the file:", self.methodDict.keys())
for m in methodNames:
methodFromFile = self.methodDict.get(m.lower())
if bool(g.getConfig("methodParametersFromFile")) and methodFromFile is not None:
outf.write(' ' + str(ElementTree.tostring(methodFromFile)))
else:
predefinedMethod = PREDEFINED_METHODS.get(m)
if predefinedMethod is None:
g.log(LOG_ERROR, "Unknown or unsupported optimization method {}".format(m))
else:
outf.write(' ' + predefinedMethod)
# finish off
outf.write('\n </Task>\n')
def prepare(self, workDir, copasiFile, startParamValues):
filename = "job{}_runner{}".format(self.job.id, self.id)
# Use separate directory for each run to avoid too many files per directory
# or at least not hit this issue too early.
# Note: there are just 65535 max files per directory on FAT32!
dirname = os.path.join(workDir, "job{}".format(self.job.id))
try:
os.mkdir(dirname)
except:
pass # may already exist, that's fine
self.inputFilename = os.path.join(dirname,
"input_" + filename + ".cps")
self.reportFilename = os.path.join(dirname,
"output_" + filename + ".log")
self.copasiFile = copasiFile
if not copasiFile.createCopy(self.inputFilename, self.reportFilename,
self.job.params, [self.methodName],
startParamValues,
self.job.areParametersChangeable):
return False
# rename the old report file, if any expected
try:
if self.generation > 1:
shutil.move(
self.reportFilename,
self.reportFilename + "_gen" + str(self.generation - 1))
else:
os.remove(self.reportFilename)
except IOError as e:
pass
except:
pass # may not exist, that's fine
copasiExe = os.path.join(COPASI_DIR, COPASI_EXECUTABLE)
if not isExecutable(copasiExe):
g.log(
LOG_ERROR,
'COPASI binary is not executable or does not exist under "' +
copasiExe + '"')
return False
args = [copasiExe, "--nologo", self.inputFilename]
self.process = process.Process(args, self)
return True
def ioGetResults(self, qs):
totalLimit = int(qs.get("totallimit", 0))
perParamLimit = int(qs.get("perparamlimit", 0))
filename = self.dumpResults(totalLimit, perParamLimit)
contents = ""
try:
with open(filename) as f:
contents = f.read()
except IOError as e:
g.log(LOG_DEBUG,
"failed to read result .csv file {}".format(filename))
except Exception as e:
g.log(LOG_INFO,
"failed to read result .csv file {}: {}".format(filename, e))
return contents
def startFromWeb(configFileName):
if not g.prepare(configFileName):
g.log(LOG_ERROR,
"Preparing from config file failed: " + configFileName)
return None
# read COPASI model file etc.
strategyManager = strategy.StrategyManager()
if not strategyManager.prepare(isDummy=False):
g.log(LOG_ERROR, "Preparing for execution failed")
return None
# start the selected parameter sweep strategy asynchronously
process.createBackgroundThread(lambda s: s.execute(), strategyManager)
return strategyManager
def dumpResults(self, totalLimit = 0, perParamLimit = 0):
if g.workDir is None:
return None
filename = g.getConfig("output.filename")
# do not allow put the results in other directories because of security reasons
if filename != os.path.basename(filename):
g.log(LOG_INFO, "output file name should not include path, ignoring all but the last element in it")
filename = os.path.basename(filename)
(name, ext) = os.path.splitext(filename)
if not ext:
ext = ".csv" # default
filename = os.path.join(g.workDir, "{}-{}{}".format(name, "-", g.taskName, ext))
with self.jobLock:
if len(self.finishedJobs) <= self.lastNumJobsDumped:
# all finished jobs already were saved, nothing to do
return filename
self.lastNumJobsDumped = len(self.finishedJobs)
allJobsByBestOfValue = []
if perParamLimit == 0:
numberOfBestCombinations = int(g.getConfig("output.numberOfBestCombinations"))
else:
numberOfBestCombinations = perParamLimit
for joblist in self.jobsByBestOfValue:
if numberOfBestCombinations:
lst = joblist[:numberOfBestCombinations]
else:
lst = joblist
for job in lst:
allJobsByBestOfValue.append(job)
allJobsByBestOfValue.sort(key=lambda x: x.getBestOfValue(), reverse=True)
allParams = self.copasiConfig["params"]
cnt = 0
with open(filename, "w") as f:
self.dumpCsvFileHeader(f)
for job in allJobsByBestOfValue:
job.dumpResults(f, allParams)
cnt += 1
if totalLimit and cnt >= totalLimit:
break
g.log(LOG_INFO, '<spacescanner>: results of finished jobs saved in "' + filename + '"')
return filename
def prepare(self, workDir, copasiFile, startParamValues):
filename = "job{}_runner{}".format(self.job.id, self.id)
# Use separate directory for each run to avoid too many files per directory
# or at least not hit this issue too early.
# Note: there are just 65535 max files per directory on FAT32!
dirname = os.path.join(workDir, "job{}".format(self.job.id))
try:
os.mkdir(dirname)
except:
pass # may already exist, that's fine
self.inputFilename = os.path.join(dirname, "input_" + filename + ".cps")
self.reportFilename = os.path.join(dirname, "output_" + filename + ".log")
self.copasiFile = copasiFile
if not copasiFile.createCopy(
self.inputFilename,
self.reportFilename,
self.job.params,
[self.methodName],
startParamValues,
self.job.areParametersChangeable,
):
return False
# rename the old report file, if any expected
try:
if self.generation > 1:
shutil.move(self.reportFilename, self.reportFilename + "_gen" + str(self.generation - 1))
else:
os.remove(self.reportFilename)
except IOError as e:
pass
except:
pass # may not exist, that's fine
copasiExe = os.path.join(COPASI_DIR, COPASI_EXECUTABLE)
if not isExecutable(copasiExe):
g.log(LOG_ERROR, 'COPASI binary is not executable or does not exist under "' + copasiExe + '"')
return False
args = [copasiExe, "--nologo", self.inputFilename]
self.process = process.Process(args, self)
return True
def totalOptimizationPotentialReached(self, numParameters):
if not self.isTOPEnabled():
return False
targetFraction = g.getConfig("optimization.targetFractionOfTOP")
# calculate the target value, looking at both config and at the job with all parameters, if any
try:
configTarget = float(g.getConfig("optimization.bestOfValue"))
except:
g.log(
LOG_INFO, "Bad bestOfValue in config: {}".format(
g.getConfig("optimization.bestOfValue")))
return False
joblist = self.jobsByBestOfValue[-1]
if joblist:
calculatedTarget = joblist[0].getBestOfValue()
else:
calculatedTarget = MIN_OF_VALUE
targetValue = max(configTarget, calculatedTarget)
if targetValue == MIN_OF_VALUE:
g.log(
LOG_DEBUG,
"TOP: no target value: {} {}".format(configTarget,
calculatedTarget))
return False
achievedValue = MIN_OF_VALUE
for joblist in self.jobsByBestOfValue[:numParameters + 1]:
if joblist:
achievedValue = max(achievedValue, joblist[0].getBestOfValue())
isReached = False
if self.topBaseline > targetValue:
isReached = True
requiredValue = targetValue
else:
requiredValue = (targetValue - self.topBaseline
) * targetFraction + self.topBaseline
isReached = achievedValue >= requiredValue
g.log(
LOG_DEBUG,
"TOP: {} parameters, {} achieved, {} required, {} target, {} configTarget, {} calculatedTarget"
.format(numParameters, achievedValue, requiredValue, targetValue,
configTarget, calculatedTarget))
if isReached:
g.log(
LOG_INFO,
"Terminating optimization at {} parameters: good-enough-value criteria reached (required {})"
.format(numParameters, requiredValue))
return True
return False
def loadParameters(self):
assert self.optimizationTask is not None
problem = self.optimizationTask.find('copasi:Problem', COPASI_NS)
if problem is None:
g.log(LOG_ERROR, "'Problem' not found in the optimization task")
return []
self.paramDict = {}
self.methodDict = {}
# Example XML structure:
# Task type="optimization"><ParameterGroup name="OptimizationItem"><Parameter> ...
for paramGroup in problem.findall('copasi:ParameterGroup', COPASI_NS):
if paramGroup.get("name").lower() == "optimizationitemlist":
for paramGroup2 in paramGroup.findall('copasi:ParameterGroup', COPASI_NS):
if paramGroup2.get("name").lower() != "optimizationitem": continue
# Example XML syntax:
# <Parameter name="ObjectCN" type="cn" value="CN=Root,Model=Galazzo1990_FermentationPathwayKinetics,Vector=Reactions[Pyruvate kinase],ParameterGroup=Parameters,Parameter=Vm6,Reference=Value"/>
for param in paramGroup2.findall('copasi:Parameter', COPASI_NS):
if param.get("name").lower() != "objectcn": continue
val = param.get("value")
if not val: continue
val = [x.split("=") for x in val.split(",")]
for (k,v) in val:
if 0:
if k.lower() == "parameter":
self.paramDict["'" + v + "'"] = paramGroup2
else:
if k.lower() == "vector":
if "[" in v and "]" in v:
v = v[v.find("[")+1:v.find("]")]
self.paramDict["'" + v + "'"] = paramGroup2
self.objectiveFunction = None
for paramText in problem.findall('copasi:ParameterText', COPASI_NS):
if paramText.get("name").lower() == "objectiveexpression":
self.objectiveFunction = paramText.text.strip()
# parse methods as well
for method in self.optimizationTask.findall('copasi:Method', COPASI_NS):
mtype = method.get("type").lower()
self.methodDict[mtype] = method
def __init__(self, line):
# input example:
# CPU time [Best Value] [Function Evaluations] [Best Parameters] maximum real part
# 0.043704 3.24353 1 ( 74.248 2.27805 ) -1.81914
self.isValid = True
self.params = []
self.cpuTime = 0.0
self.ofValue = MIN_OF_VALUE
self.numOfEvaluations = 0
self.maxRealPart = 0.0
line = line.strip()
if not line:
self.isValid = False
return
if "\t" in line:
numbers = line.split("\t")
else:
numbers = line.split(" ")
try:
self.cpuTime = float(numbers[0])
self.ofValue = float(numbers[1])
# check for NaN and +inf, but allow -inf, as it's
# sometimes returned as the "no solution found" value
if math.isnan(self.ofValue) or \
(math.isinf(self.ofValue) and self.ofValue > 0.0):
# XXX: something went wrong, what's the best action?
g.log(
LOG_ERROR,
"invalid objective function value {}, using -infinity instead"
.format(self.ofValue))
self.ofValue = MIN_OF_VALUE
self.numOfEvaluations = int(numbers[2])
self.maxRealPart = float(numbers[-1])
# param value list starts with "(", finishes with ")"
for i in range(4, len(numbers) - 2):
self.params.append(float(numbers[i]))
except ValueError as e:
g.log(LOG_DEBUG, "value error {} in line".format(e))
g.log(LOG_DEBUG, line)
except:
g.log(LOG_DEBUG,
"unexpected error {} in line".format(sys.exc_info()[0]))
g.log(LOG_DEBUG, line)
self.isValid = False
def writeParam(self, outf, param, startParamValues, areParametersChangeable):
xml = self.paramDict[param]
if areParametersChangeable:
if startParamValues is None or param not in startParamValues:
# no need to preprocess; write directly back in the file
outf.write(' ' + str(ElementTree.tostring(xml)))
return
else:
if startParamValues is None:
startParamValues = {}
for sub in xml.iterfind("*", COPASI_NS):
if "ParameterGroup" in sub.tag: continue
if sub.get("name").lower() == "startvalue" and param not in startParamValues:
# read it directly from the file
try:
v = sub.get("value")
startParamValues[param] = float(v)
except:
g.log(LOG_ERROR, "Getting start value of a parameter " + param + " failed:" + sub.get("value"))
outf.write(' <ParameterGroup name="OptimizationItem">\n')
for sub in xml.iterfind("*", COPASI_NS):
if "ParameterGroup" in sub.tag: continue
if sub.get("name").lower() == "startvalue" \
and startParamValues is not None \
and param in startParamValues:
outf.write(' <Parameter name="StartValue" type="float" value="{}"/>\n'.format(startParamValues[param]))
continue
if sub.get("name").lower() in ["lowerbound", "upperbound"] \
and not areParametersChangeable \
and startParamValues is not None \
and param in startParamValues:
# unchangeable; set the bounds to the start value
val = startParamValues[param]
outf.write(' <Parameter name="{}" type="cn" value="{}"/>\n'.format(sub.get("name"), val))
continue
# by default, just write back whatever was in the file
outf.write(' ' + str(ElementTree.tostring(sub)))
outf.write(' </ParameterGroup>\n')
def executeWebserver(strategyManager):
try:
port = int(g.getConfig("web.port"))
g.log(LOG_INFO,
"<spacescanner>: starting webserver, port: " + str(port))
server = webserver.InterruptibleHTTPServer(('', port),
webserver.HttpServerHandler)
server.strategyManager = strategyManager
# report ok and enter the main loop
g.log(
LOG_DEBUG,
"<spacescanner>: webserver started, listening to port {}".format(
port))
server.serve_forever()
except Exception as e:
g.log(LOG_ERROR, "<spacescanner>: exception occurred in webserver:")
g.log(LOG_ERROR, str(e))
g.log(LOG_ERROR, traceback.format_exc())
sys.exit(1)
sys.exit(0)
def run(self):
#g.log(LOG_DEBUG, "Run subprocess: " + " ".join(self.args) + "\n")
retcode = -1
try:
with open(os.devnull, 'w') as fp:
self.process = Popen(self.args, stdout = fp, stderr = STDOUT)
self.psutilProcess = psutil.Process(self.process.pid)
while self.process.poll() is None:
if self.runner.shouldTerminate():
self.process.kill()
time.sleep(Process.POLL_INTERVAL)
retcode = self.process.returncode
except Exception as e:
g.log(LOG_ERROR, "run subprocess exception: " + str(e))
except:
g.log(LOG_ERROR, "run subprocess unexpected error: {}".format(sys.exc_info()[0]))
#g.log(LOG_DEBUG, "subprocess done, return code: " + str(retcode))
return retcode
def serve_forever(self, poll_interval = 0.5):
"""Handle one request at a time until shutdown.
Polls for shutdown every poll_interval seconds. Ignores
self.timeout. If you need to do periodic tasks, do them in
another thread.
"""
self._BaseServer__is_shut_down.clear()
try:
while not self._BaseServer__shutdown_request:
# XXX: Consider using another file descriptor or
# connecting to the socket to wake this up instead of
# polling. Polling reduces our responsiveness to a
# shutdown request and wastes cpu at all other times.
r, w, e = select.select([self], [], [], poll_interval)
if self in r:
self._handle_request_noblock()
except Exception as e:
g.log(LOG_ERROR, "base server exception:")
g.log(LOG_ERROR, str(e))
g.log(LOG_ERROR, traceback.format_exc())
finally:
self._BaseServer__shutdown_request = False
self._BaseServer__is_shut_down.set()
if os.name == "posix":
# kill the process to make sure it exits
os.kill(os.getpid(), signal.SIGKILL)
def serve_forever(self, poll_interval=0.5):
"""Handle one request at a time until shutdown.
Polls for shutdown every poll_interval seconds. Ignores
self.timeout. If you need to do periodic tasks, do them in
another thread.
"""
self._BaseServer__is_shut_down.clear()
try:
while not self._BaseServer__shutdown_request:
# XXX: Consider using another file descriptor or
# connecting to the socket to wake this up instead of
# polling. Polling reduces our responsiveness to a
# shutdown request and wastes cpu at all other times.
r, w, e = select.select([self], [], [], poll_interval)
if self in r:
self._handle_request_noblock()
except Exception as e:
g.log(LOG_ERROR, "base server exception:")
g.log(LOG_ERROR, str(e))
g.log(LOG_ERROR, traceback.format_exc())
finally:
self._BaseServer__shutdown_request = False
self._BaseServer__is_shut_down.set()
if os.name == "posix":
# kill the process to make sure it exits
os.kill(os.getpid(), signal.SIGKILL)
def __init__(self, line):
# input example:
# CPU time [Best Value] [Function Evaluations] [Best Parameters] maximum real part
# 0.043704 3.24353 1 ( 74.248 2.27805 ) -1.81914
self.isValid = True
self.params = []
self.cpuTime = 0.0
self.ofValue = MIN_OF_VALUE
self.numOfEvaluations = 0
self.maxRealPart = 0.0
line = line.strip()
if not line:
self.isValid = False
return
if "\t" in line:
numbers = line.split("\t")
else:
numbers = line.split(" ")
try:
self.cpuTime = float(numbers[0])
self.ofValue = float(numbers[1])
# check for NaN and +inf, but allow -inf, as it's
# sometimes returned as the "no solution found" value
if math.isnan(self.ofValue) or (math.isinf(self.ofValue) and self.ofValue > 0.0):
# XXX: something went wrong, what's the best action?
g.log(LOG_ERROR, "invalid objective function value {}, using -infinity instead".format(self.ofValue))
self.ofValue = MIN_OF_VALUE
self.numOfEvaluations = int(numbers[2])
self.maxRealPart = float(numbers[-1])
# param value list starts with "(", finishes with ")"
for i in range(4, len(numbers) - 2):
self.params.append(float(numbers[i]))
except ValueError as e:
g.log(LOG_DEBUG, "value error {} in line".format(e))
g.log(LOG_DEBUG, line)
except:
g.log(LOG_DEBUG, "unexpected error {} in line".format(sys.exc_info()[0]))
g.log(LOG_DEBUG, line)
self.isValid = False
def totalOptimizationPotentialReached(self, numParameters):
if not self.isTOPEnabled():
return False
optimality = g.getConfig("optimization.optimalityRelativeError")
# calculate the target value, looking at both config and at the job with all parameters, if any
try:
configTarget = float(g.getConfig("optimization.bestOfValue"))
except:
g.log(LOG_INFO, "Bad bestOfValue in config: {}".format(g.getConfig("optimization.bestOfValue")))
return False
joblist = self.jobsByBestOfValue[-1]
if joblist:
calculatedTarget = joblist[0].getBestOfValue()
else:
calculatedTarget = MIN_OF_VALUE
targetValue = max(configTarget, calculatedTarget)
if targetValue == MIN_OF_VALUE:
return False
achievedValue = MIN_OF_VALUE
for joblist in self.jobsByBestOfValue[:numParameters + 1]:
if joblist:
achievedValue = max(achievedValue, joblist[0].getBestOfValue())
proportion = 1.0 - float(optimality)
isReached = False
if self.topBaseline > targetValue:
isReached = True
requiredValue = targetValue
else:
requiredValue = (targetValue - self.topBaseline) * proportion + self.topBaseline
isReached = achievedValue >= requiredValue
if isReached:
g.log(LOG_INFO, "terminating optimization at {} parameters: good-enough-value criteria reached (required {})".format(numParameters, requiredValue))
return True
return False
def create(specification, strategy):
if "type" not in specification:
return None
isReverse = False
start = 0; end = 0
if "range" in specification:
numParams = len(strategy.copasiConfig["params"])
range = specification["range"]
start = range[0]
if len(range) >= 2:
end = range[1]
else:
end = start
if start < 0: # negative range
start = numParams - start
if end < 0:
end = numParams - end
if start == 0 or end == 0:
g.log(LOG_ERROR, "parameter selection ranges must contain numbers in the range [1 .. n]")
return None
if specification["type"] == "full-set":
x = ParamSelectionFullSet(strategy)
elif specification["type"] == "zero":
x = ParamSelectionZero(strategy)
elif specification["type"] == "explicit":
# use a singleton instance
if ParamSelection.instanceOfExplicit is None:
ParamSelection.instanceOfExplicit = ParamSelectionExplicit(strategy)
x = ParamSelection.instanceOfExplicit
if specification.get("parameters"):
names = []
for p in specification["parameters"]:
p = "'" + p + "'"
if p not in strategy.copasiConfig["params"]:
g.log(LOG_ERROR, "'explicit' parameter range contains nonexistent parameter name {}".format(p))
return None
names.append(p)
x.explicitParameterSets.append(names)
else:
g.log(LOG_ERROR, "'explicit' parameter range must contain a list of parameter names")
return None
elif specification["type"] == "exhaustive":
x = ParamSelectionExhaustive(strategy, start, end)
elif specification["type"] == "greedy":
newstart = min(start, end)
newend = max(start, end)
x = ParamSelectionGreedy(strategy, newstart, newend)
elif specification["type"] == "greedy-reverse":
newstart = max(start, end)
newend = min(start, end)
x = ParamSelectionGreedyReverse(strategy, newstart, newend)
else:
return None
return x
def read(self, filename):
if not isReadable(filename):
g.log(
LOG_ERROR,
"error while loading COPASI model: file not found or not readable"
)
return False
self.optimizationTask = None
self.xmlroot = ElementTree.parse(filename).getroot()
# Example XML structure:
# <xml><ListOfTasks><Task type="optimization">...
for tasklist in self.xmlroot.findall('copasi:ListOfTasks', COPASI_NS):
for task in tasklist.findall('copasi:Task', COPASI_NS):
if task.get("type").lower() == "optimization":
self.optimizationTask = task
if self.optimizationTask is None:
g.log(
LOG_ERROR,
"error while loading COPASI model: optimization task not found in COPASI file"
)
return False
self.loadParameters()
if len(self.paramDict) == 0:
g.log(
LOG_ERROR,
"error while loading COPASI model: optimization parameters not defined in COPASI file"
)
return False
if not self.objectiveFunction:
g.log(
LOG_ERROR,
"error while loading COPASI model: objective function not defined in COPASI file"
)
return False
return True
def checkReport(self, hasTerminated, now):
assert self.isActive
if not hasTerminated:
if self.lastReportCheckTime is not None and now - self.lastReportCheckTime < MIN_REPORT_CHECK_INTERVAL:
# too soon, skip
return True
if now - self.startTime < 3.0:
# too soon after start (copasi may not be launched yet), return
return True
self.lastReportCheckTime = now
oldOfValue = self.ofValue
try:
st = os.stat(self.reportFilename)
if self.lastReportModificationTime is None or st.st_mtime > self.lastReportModificationTime:
self.lastReportModificationTime = st.st_mtime
with open(self.reportFilename, "r") as f:
self.stats.isValid = False
inValues = False
for line in f:
if startsWith(line, "CPU time"):
inValues = True
continue
if not inValues:
continue
if startsWith(line, "Optimization Result"):
break
si = StatsItem(line)
if si.isValid:
self.stats = si
if self.stats.isValid:
self.ofValue = self.getLastStats().ofValue
if oldOfValue != self.ofValue:
g.log(LOG_INFO, "{}: new OF value {}".format(self.getName(), self.ofValue))
# Check for CPU time end condition using the report file
self.currentCpuTime = self.getLastStats().cpuTime
else:
# no report file update; check for CPU time end condition using OS measurements
if not hasTerminated:
try:
self.currentCpuTime = self.process.getCpuTime()
except psutil.NoSuchProcess:
pass # has already quit
except OSError as e:
g.log(LOG_ERROR, "accessing report file {} failed: {}".format(self.reportFilename, os.strerror(e.errno)))
except IOError as e:
g.log(LOG_ERROR, "parsing report file {} failed: {}".format(self.reportFilename, os.strerror(e.errno)))
if not hasTerminated and not self.terminationReason:
g.log(LOG_DEBUG, "checked {}, CPU time: {}".format(self.getName(), self.currentCpuTime))
# XXX: hardcoded "slow" method names
if self.methodName not in ["ScatterSearch", "SimulatedAnnealing"]:
# XXX: reuse consensus time for this
maxTimeWithNoValue = float(g.getConfig("optimization.consensusMinDurationSec")) + 10.0
if self.job.timeDiffExceeded(now - self.startTime, maxTimeWithNoValue) and (
not self.stats.isValid or self.ofValue == MIN_OF_VALUE
):
g.log(
LOG_DEBUG,
"terminating {}: no value found in CPU time: {}".format(self.getName(), self.currentCpuTime),
)
self.terminationReason = TERMINATION_REASON_CPU_TIME_LIMIT
if oldOfValue != self.ofValue:
# the OF value was updated
return True
return False # the OF value was not updated
def create(specification, strategy):
if "type" not in specification:
return None
isReverse = False
start = 0
end = 0
if "range" in specification:
numParams = len(strategy.copasiConfig["params"])
range = specification["range"]
start = range[0]
if len(range) >= 2:
end = range[1]
else:
end = start
if start < 0: # negative range
start = numParams - start
if end < 0:
end = numParams - end
if start == 0 or end == 0:
g.log(
LOG_ERROR,
"parameter selection ranges must contain numbers in the range [1 .. n]"
)
return None
if specification["type"] == "full-set":
x = ParamSelectionFullSet(strategy)
elif specification["type"] == "zero":
x = ParamSelectionZero(strategy)
elif specification["type"] == "explicit":
# use a singleton instance
if ParamSelection.instanceOfExplicit is None:
ParamSelection.instanceOfExplicit = ParamSelectionExplicit(
strategy)
x = ParamSelection.instanceOfExplicit
if specification.get("parameters"):
names = []
for p in specification["parameters"]:
p = "'" + p + "'"
if p not in strategy.copasiConfig["params"]:
g.log(
LOG_ERROR,
"'explicit' parameter range contains nonexistent parameter name {}"
.format(p))
return None
names.append(p)
x.explicitParameterSets.append(names)
else:
g.log(
LOG_ERROR,
"'explicit' parameter range must contain a list of parameter names"
)
return None
elif specification["type"] == "exhaustive":
x = ParamSelectionExhaustive(strategy, start, end)
elif specification["type"] == "greedy":
newstart = min(start, end)
newend = max(start, end)
x = ParamSelectionGreedy(strategy, newstart, newend)
elif specification["type"] == "greedy-reverse":
newstart = max(start, end)
newend = min(start, end)
x = ParamSelectionGreedyReverse(strategy, newstart, newend)
else:
return None
return x
def execute(self):
parameterSelections = []
if g.getConfig("restartOnFile"):
# Guess which parameters have not been optimized yet based on the .csv result file
filename = g.getConfig("restartOnFile").replace("@SELF@", SELF_PATH)
parameterSets = getNonconvergedResults(filename)
for ps in parameterSets:
spec = {"type" : "explicit", "parameters" : ps}
x = ParamSelection.create(spec, self)
if x not in parameterSelections:
parameterSelections.append(x)
elif g.getConfig("parameters"):
# Deal with TOP, if required
if self.isTOPEnabled():
# add the "zero" at the start
g.log(LOG_INFO, "optimizing for zero parameters initially to find the baseline")
spec = {"type" : "zero"}
parameterSelections.append(ParamSelection.create(spec, self))
# The take the other sets from the file
for spec in g.getConfig("parameters"):
x = ParamSelection.create(spec, self)
if x is None:
g.log(LOG_ERROR, "invalid parameter specification: {}".format(ENC.encode(spec)))
continue
if x not in parameterSelections:
parameterSelections.append(x)
else:
# add the default optimization target: all parameters
g.log(LOG_INFO, "optimizing only for all parameters")
spec = {"type" : "full-set"}
parameterSelections.append(ParamSelection.create(spec, self))
numCombinations = 0
for sel in parameterSelections:
numCombinations += sel.getNumCombinations()
g.log(LOG_INFO, "total {} parameter combination(s) to try out, parameters: {}".format(numCombinations, " ".join(self.copasiConfig["params"])))
g.log(LOG_INFO, "methods enabled: '" + "' '".join(g.getConfig("copasi.methods")) + "'")
parameterSelections.sort(key = lambda x: x.getSortOrder())
for sel in parameterSelections:
areParametersChangeable = sel.type != PARAM_SEL_ZERO
for params in sel.getParameterSets():
g.log(LOG_DEBUG, "made a new pool of {} jobs".format(len(params)))
pool = jobpool.JobPool(self, params, areParametersChangeable)
with self.jobLock:
self.activeJobPool = pool
pool.start()
while True:
time.sleep(1.0)
if self.doQuitFlag:
return True
try:
pool.refresh()
except Exception as e:
g.log(LOG_INFO, "Exception while refreshing active joob pool status, terminating the pool: {}".format(e))
self.finishActivePool()
break
if pool.isDepleted():
self.finishActivePool()
break
return True
def checkReports(self):
# if no runners are active, quit
if not any([r.isActive for r in self.runners]):
if self.convergenceTime is not None:
minAbsoluteTime = float(g.getConfig("optimization.consensusMinDurationSec"))
# XXX: do not check the relative time here
if self.hasConsensus() and self.timeDiffExceeded(time.time() - self.convergenceTime, minAbsoluteTime):
# count COPASI termination as consensus in this case
# XXX: note that this does *not* overwrite "time limit exceeded" exit code!
for r in self.runners:
if r.terminationReason == TERMINATION_REASON_COPASI_FINISHED:
r.terminationReason = TERMINATION_REASON_CONSENSUS
# switch the methods if required
self.decideTermination()
return
if not all([r.isActive for r in self.runners]):
# some but not all have quit; quit the others with "stagnation"
# (not technically true, but the best match from the existing codes)
for r in self.runners:
if r.isActive and not r.terminationReason:
r.terminationReason = TERMINATION_REASON_STAGNATION
return
numActiveRunners = 0
now = time.time()
cpuTimeLimit = float(g.getConfig("optimization.timeLimitSec"))
maxCpuTime = 0
anyUpdated = False
with runner.reportLock:
for r in self.runners:
if r.isActive:
numActiveRunners += 1
if r.checkReport(hasTerminated=False, now=now):
# the runner updated the OF time
self.lastOfUpdateTime = now
anyUpdated = True
maxCpuTime = max(maxCpuTime, r.currentCpuTime)
if not self.areParametersChangeable:
# it is simple; as soon as the first value is read, return
if anyUpdated:
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CONSENSUS
return
consensusReached = self.hasConsensus()
if all([r.terminationReason for r in self.runners]):
return
# use the old CPU time as a basis
maxCpuTime += sum(self.oldCpuTimes)
doKillOnTimeLimit = maxCpuTime >= cpuTimeLimit and not consensusReached
if doKillOnTimeLimit:
# kill all jobs immediately
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CPU_TIME_LIMIT
g.log(
LOG_INFO,
"terminating {}: CPU time limit exceeded ({} vs. {})".format(
r.getName(), r.currentCpuTime, cpuTimeLimit
),
)
return
# check if the runs have reached consensus
if consensusReached:
if self.convergenceTime is None:
g.log(LOG_DEBUG, self.getName() + ": reached consensus, waiting for guard time before termination")
self.convergenceTime = now
self.convergenceValue = min([r.ofValue for r in self.runners])
# if the runners have converged for long enough time, quit
else:
timeConverged = time.time() - self.convergenceTime
minAbsoluteTime = float(g.getConfig("optimization.consensusMinDurationSec"))
minRelativeTime = (time.time() - self.startTime) * float(
g.getConfig("optimization.consensusMinProportionalDuration")
)
if self.timeDiffExceeded(timeConverged, minAbsoluteTime) and timeConverged > minRelativeTime:
g.log(LOG_INFO, "terminating {}: consensus reached".format(self.getName()))
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CONSENSUS
self.convergenceTime = now
return # do not check other criteria
else:
# reset the timer
self.convergenceTime = None
# check for stagnation's time limit
timeStagnated = time.time() - self.lastOfUpdateTime
maxAbsoluteTime = float(g.getConfig("optimization.stagnationMaxDurationSec"))
maxRelativeTime = (time.time() - self.startTime) * float(
g.getConfig("optimization.stagnationMaxProportionalDuration")
)
if self.timeDiffExceeded(timeStagnated, maxAbsoluteTime) and timeStagnated > maxRelativeTime:
# stagnation detected
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_STAGNATION
g.log(
LOG_INFO,
"terminating {}: Optimization stagnated (did not produce new results) for {} seconds".format(
self.getName(), timeStagnated
),
)
return
# We will continue. Check if load balancing needs to be done
if now - self.lastBalanceTime >= LOAD_BALANCE_INTERVAL:
self.lastBalanceTime = now
if numActiveRunners > self.maxCores:
# not converged yet + too many active; limit some runners
cpuTimes = [(r.currentCpuTime, r) for r in self.runners if r.isActive]
cpuTimes.sort()
# continue first `maxCores` runners, suspend the rest
resumeRunners = cpuTimes[: self.maxCores]
suspendRunners = cpuTimes[self.maxCores :]
for _, j in resumeRunners:
j.suspend(False)
for _, j in suspendRunners:
j.suspend(True)
else:
for r in self.runners:
r.suspend(False)
def doQuit():
g.log(LOG_ERROR, "Terminating...")
g.doQuit = True
def do_GET(self):
o = urlparse(self.path)
qs = parse_qs(o.query)
isJSON = True
contentType = "text/plain"
sm = InterruptibleHTTPServer.serverInstance.strategyManager
if o.path == '/index.html' or o.path == "/":
isJSON = False
contentType = "text/html"
try:
with open(os.path.join(SELF_PATH, "web", "index.html")) as f:
response = f.read()
except:
self.serveError(qs)
return
elif o.path[-4:] == ".css":
isJSON = False
contentType = "text/css"
try:
with open(os.path.join(SELF_PATH, "web", o.path[1:])) as f:
response = f.read()
except:
self.serveError(qs)
return
elif o.path[-3:] == ".js":
isJSON = False
contentType = "application/js"
try:
with open(os.path.join(SELF_PATH, "web", o.path[1:])) as f:
response = f.read()
except:
self.serveError(qs)
return
elif o.path[-4:] == ".png":
isJSON = False
contentType = "image/png"
try:
with open(os.path.join(SELF_PATH, "web", o.path[1:])) as f:
response = f.read()
except:
self.serveError(qs)
return
elif o.path[-4:] == ".jpg":
isJSON = False
contentType = "image/jpg"
try:
with open(os.path.join(SELF_PATH, "web", o.path[1:])) as f:
response = f.read()
except:
self.serveError(qs)
return
elif "font" in o.path:
isJSON = False
contentType = "font/opentype"
try:
with open(os.path.join(SELF_PATH, "web", o.path[1:])) as f:
response = f.read()
except:
self.serveError(qs)
return
elif o.path == '/status':
# active jobs
response = {
"isActive": sm.isActive(),
"isExecutable": sm.isExecutable,
"totalNumJobs": sm.getTotalNumJobs(),
"totalNumParams": sm.getTotalNumParams(),
"resultsPresent": sm.getNumFinishedJobs() > 0
}
elif o.path == '/allstatus':
response = sm.ioGetAllJobs(qs)
elif o.path == '/activestatus':
response = sm.ioGetActiveJobs(qs)
elif o.path[:4] == '/job':
# specific job
response = sm.ioGetJob(qs, o.path[5:])
elif o.path[:7] == '/config':
response = sm.ioGetConfig(qs)
print("config=", response)
elif o.path[:8] == '/results':
response = sm.ioGetResults(qs)
isJSON = False # the results are in .csv format
contentType = "text/csv"
elif o.path == '/stopall':
response = sm.ioStopAll(qs)
elif o.path[:5] == '/stop':
response = sm.ioStop(qs, o.path[6:])
elif o.path == '/terminate':
g.log(LOG_ERROR, "Terminating upon user request")
response = {"status": "ok"}
threading.Timer(1.0, doQuit).start()
else:
self.serveError(qs)
return
if isJSON:
response = ENC.encode(response)
self.send_response(200)
self.sendDefaultHeaders(response, isJSON, contentType)
self.end_headers()
self.serveBody(response, qs)
def serializeOptimizationTask(self, reportFilename, outf, parameters,
methodNames, startParamValues,
areParametersChangeable):
# Note: 'scheduled' is always set to true, as is 'update model' to save the final parameter values in the .cps file.
outf.write(
' <Task key="{}" name="Optimization" type="optimization" scheduled="true" updateModel="true">\n'
.format(self.optimizationTask.get("key")))
# 1. Fix report target file name
# <Report reference="Report_10" target="./report.log" append="1"/>
report = self.optimizationTask.find('copasi:Report', COPASI_NS)
if report is not None:
report.set("target", reportFilename)
report.set("reference", "optimization_report")
outf.write(' ' + str(ElementTree.tostring(report)))
# 2. Include only required parameters
problem = self.optimizationTask.find('copasi:Problem', COPASI_NS)
outf.write(' <Problem>\n')
for elem in problem.iterfind("*", COPASI_NS):
if "Problem" in elem.tag: continue
if "ParameterGroup" not in elem.tag or elem.get(
"name").lower() != "optimizationitemlist":
if "Parameter" in elem.tag and elem.get(
"name").lower() == "randomize start values":
# never randomize them
outf.write(
' <Parameter name="Randomize Start Values" type="bool" value="0"/>\n'
)
else:
outf.write(' ' + str(ElementTree.tostring(elem)))
continue
#print("parameters in the file:", self.paramDict.keys())
outf.write(' <ParameterGroup name="OptimizationItemList">\n')
for p in parameters:
if p in self.paramDict:
self.writeParam(outf, p, startParamValues,
areParametersChangeable)
outf.write(' </ParameterGroup>\n')
outf.write(' </Problem>\n')
# 3. Include only required methods
#print("methods in the file:", self.methodDict.keys())
for m in methodNames:
methodFromFile = self.methodDict.get(m.lower())
if bool(g.getConfig("methodParametersFromFile")
) and methodFromFile is not None:
outf.write(' ' +
str(ElementTree.tostring(methodFromFile)))
else:
predefinedMethod = PREDEFINED_METHODS.get(m)
if predefinedMethod is None:
g.log(
LOG_ERROR,
"Unknown or unsupported optimization method {}".format(
m))
else:
outf.write(' ' + predefinedMethod)
# finish off
outf.write('\n </Task>\n')
def checkReport(self, hasTerminated, now):
assert self.isActive
if not hasTerminated:
if self.lastReportCheckTime is not None \
and now - self.lastReportCheckTime < MIN_REPORT_CHECK_INTERVAL:
# too soon, skip
return False
if now - self.startTime < 3.0:
# too soon after start (copasi may not be launched yet), return
return False
self.lastReportCheckTime = now
oldOfValue = self.ofValue
try:
st = os.stat(self.reportFilename)
if self.lastReportModificationTime is None \
or st.st_mtime > self.lastReportModificationTime:
self.lastReportModificationTime = st.st_mtime
with open(self.reportFilename, "r") as f:
self.stats.isValid = False
inValues = False
for line in f:
if startsWith(line, "CPU time"):
inValues = True
continue
if not inValues: continue
if startsWith(line, "Optimization Result"):
break
si = StatsItem(line)
if si.isValid:
self.stats = si
if self.stats.isValid:
self.ofValue = self.getLastStats().ofValue
if oldOfValue != self.ofValue:
g.log(
LOG_INFO, "{}: new OF value {}".format(
self.getName(), self.ofValue))
# Check for CPU time end condition using the report file
reportCpuTime = self.getLastStats().cpuTime
if self.currentCpuTime < reportCpuTime:
self.currentCpuTime = reportCpuTime
else:
# no report file update; check for CPU time end condition using OS measurements
if not hasTerminated:
try:
t = self.process.getCpuTime()
except psutil.NoSuchProcess:
pass # has already quit
else:
self.currentCpuTime = t
except OSError as e:
g.log(
LOG_ERROR, "accessing report file {} failed: {}".format(
self.reportFilename, os.strerror(e.errno)))
except IOError as e:
g.log(
LOG_ERROR, "parsing report file {} failed: {}".format(
self.reportFilename, os.strerror(e.errno)))
if oldOfValue != self.ofValue:
# the OF value was updated
return True
return False # the OF value was not updated
def getBestParameters(self, k):
joblist = self.jobsByBestOfValue[k]
if not joblist:
g.log(LOG_ERROR, "Parameter ranges are invalid: best value of {} parameters requested, but no jobs finished".format(k))
return None
return joblist[0].params
def decideTermination(self):
continuableReasons = [TERMINATION_REASON_STAGNATION,
TERMINATION_REASON_COPASI_FINISHED]
if not any([r.terminationReason in continuableReasons for r in self.runners]):
# all terminated with consensus, because asked by the user, or with time limit
self.pool.finishJob(self)
return
# So we have at least one termination with either:
# a) the stagnation limit was exceeded, or
# b) Copasi stopped without consensus.
# Actions now:
# 1) if no solution found: use a fallback method;
# 2) else switch to the next method;
# 3) if no more methods are available, quit.
if self.currentMethod in self.fallbackMethods:
# remove the already-used method to avoid infinite looping between methods
self.fallbackMethods.remove(self.currentMethod)
assert (self.currentMethod in self.methods)
self.methods.remove(self.currentMethod)
anyNotFound = any([math.isinf(r.ofValue) for r in self.runners])
if anyNotFound or self.isUsingFallback:
if len(self.fallbackMethods) == 0:
if anyNotFound:
g.log(LOG_INFO, "terminating {}: failed to evaluate the objective function".format(self.getName()))
else:
g.log(LOG_INFO, "terminating {}: all fallback methods exhausted without reaching consensus".format(self.getName()))
self.pool.finishJob(self)
return
self.pastMethods.append(self.currentMethod)
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.fallbackMethods)
else:
self.currentMethod = self.fallbackMethods[0]
# make sure the fallback methods are also in methods
if self.currentMethod not in self.methods:
self.methods.append(self.currentMethod)
g.log(LOG_INFO, "switching {} to a fallback method {}".format(self.getName(), self.currentMethod))
self.isUsingFallback = True
# switch to the fallback method
if not self.createRunners():
self.pool.finishJob(self)
return
if len(self.methods) == 0:
g.log(LOG_INFO, "terminating {}: all methods exhausted without reaching consensus".format(self.getName()))
self.pool.finishJob(self)
return
# go for the next method
self.pastMethods.append(self.currentMethod)
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.methods)
else:
self.currentMethod = self.methods[0]
g.log(LOG_INFO, "switching {} to the next method {}".format(
self.getName(), self.currentMethod))
if not self.createRunners():
self.pool.finishJob(self)
return
def log_message(self, format, *args):
g.log(LOG_DEBUG, "%s - - [%s] %s" %
(self.client_address[0],
self.log_date_time_string(),
format % args))
def decideTermination(self):
continuableReasons = [TERMINATION_REASON_STAGNATION, TERMINATION_REASON_COPASI_FINISHED]
if not any([r.terminationReason in continuableReasons for r in self.runners]):
# all terminated with consensus, because asked by the user, or with time limit
self.pool.finishJob(self)
return
# So we have at least one termination with either:
# a) the stagnation limit was exceeded, or
# b) Copasi stopped without consensus.
# Actions now:
# 1) if no solution found: use a fallback method;
# 2) else switch to the next method;
# 3) if no more methods are available, quit.
if self.currentMethod in self.fallbackMethods:
# remove the already-used method to avoid infinite looping between methods
self.fallbackMethods.remove(self.currentMethod)
assert self.currentMethod in self.methods
self.methods.remove(self.currentMethod)
anyNotFound = any([math.isinf(r.ofValue) for r in self.runners])
if anyNotFound or self.isUsingFallback:
if len(self.fallbackMethods) == 0:
if anyNotFound:
g.log(LOG_INFO, "terminating {}: failed to evaluate the objective function".format(self.getName()))
else:
g.log(
LOG_INFO,
"terminating {}: all fallback methods exhausted without reaching consensus".format(
self.getName()
),
)
self.pool.finishJob(self)
return
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.fallbackMethods)
else:
self.currentMethod = self.fallbackMethods[0]
# make sure the fallback methods are also in methods
if self.currentMethod not in self.methods:
self.methods.append(self.currentMethod)
g.log(LOG_INFO, "switching {} to a fallback method {}".format(self.getName(), self.currentMethod))
self.isUsingFallback = True
# switch to the fallback method
if not self.createRunners():
self.pool.finishJob(self)
return
if len(self.methods) == 0:
g.log(LOG_INFO, "terminating {}: all methods exhausted without reaching consensus".format(self.getName()))
self.pool.finishJob(self)
return
# go for the next method
if bool(g.getConfig("optimization.randomizeMethodSelection")):
self.currentMethod = random.choice(self.methods)
else:
self.currentMethod = self.methods[0]
g.log(LOG_INFO, "switching {} to the next method {}".format(self.getName(), self.currentMethod))
if not self.createRunners():
self.pool.finishJob(self)
return
def log_message(self, format, *args):
g.log(
LOG_DEBUG,
"%s - - [%s] %s" % (self.client_address[0],
self.log_date_time_string(), format % args))
def checkReports(self):
# if no runners are active, quit
if not any([r.isActive for r in self.runners]):
if self.convergenceTime is not None:
minAbsoluteTime = float(g.getConfig("optimization.consensusDelaySec"))
# XXX: do not check the relative time here
if self.hasConsensus() and self.timeDiffExceeded(time.time() - self.convergenceTime, minAbsoluteTime):
# count COPASI termination as consensus in this case
# XXX: note that this does *not* overwrite "time limit exceeded" exit code!
for r in self.runners:
if r.terminationReason == TERMINATION_REASON_COPASI_FINISHED:
r.terminationReason = TERMINATION_REASON_CONSENSUS
# switch the methods if required
self.decideTermination()
return
if not all([r.isActive for r in self.runners]):
# some but not all have quit; quit the others with "stagnation"
# (not technically true, but the best match from the existing codes)
for r in self.runners:
if r.isActive and not r.terminationReason:
r.terminationReason = TERMINATION_REASON_STAGNATION
return
numActiveRunners = 0
now = time.time()
cpuTimeLimit = float(g.getConfig("optimization.timeLimitSec"))
maxCpuTime = 0
anyUpdated = False
with runner.reportLock:
for r in self.runners:
if r.isActive:
numActiveRunners += 1
if r.checkReport(hasTerminated = False, now = now):
# the runner updated the OF time
self.lastOfUpdateTime = now
anyUpdated = True
maxCpuTime = max(maxCpuTime, r.currentCpuTime)
if not self.areParametersChangeable:
# it is simple; as soon as the first value is read, return
if anyUpdated:
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CONSENSUS
return
consensusReached = self.hasConsensus()
if all([r.terminationReason for r in self.runners]):
return
# use the old CPU time as a basis
maxCpuTime += sum(self.oldCpuTimes)
doKillOnTimeLimit = maxCpuTime >= cpuTimeLimit and not consensusReached
if doKillOnTimeLimit:
# kill all jobs immediately
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CPU_TIME_LIMIT
g.log(LOG_INFO, "terminating {}: CPU time limit exceeded ({} vs. {})".format(
r.getName(), maxCpuTime, cpuTimeLimit))
return
# check if the runs have reached consensus
if consensusReached:
if self.convergenceTime is None:
g.log(LOG_DEBUG, self.getName() + ": reached consensus, waiting for guard time before termination")
self.convergenceTime = now
self.convergenceValue = min([r.ofValue for r in self.runners])
# if the runners have converged for long enough time, quit
else:
timeConverged = time.time() - self.convergenceTime
minAbsoluteTime = float(g.getConfig("optimization.consensusDelaySec"))
minRelativeTime = (time.time() - self.startTime) * float(g.getConfig("optimization.consensusProportionalDelay"))
if self.timeDiffExceeded(timeConverged, minAbsoluteTime) and timeConverged > minRelativeTime:
g.log(LOG_INFO, "terminating {}: consensus reached".format(self.getName()))
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_CONSENSUS
self.convergenceTime = now
return # do not check other criteria
else:
# reset the timer
self.convergenceTime = None
# check for stagnation's time limit
timeStagnated = time.time() - self.lastOfUpdateTime
maxAbsoluteTime = float(g.getConfig("optimization.stagnationDelaySec"))
maxRelativeTime = (time.time() - self.startTime) * float(g.getConfig("optimization.stagnationProportionalDelay"))
# XXX: specialcase for the non-parameters job: quit it quite quickly (in 10 seconds for each method)
if not self.areParametersChangeable:
maxAbsoluteTime = 10.0
maxRelativeTime = 0.0
if self.timeDiffExceeded(timeStagnated, maxAbsoluteTime) and timeStagnated > maxRelativeTime:
# stagnation detected
for r in self.runners:
if r.isActive:
r.terminationReason = TERMINATION_REASON_STAGNATION
g.log(LOG_INFO, "terminating {}: Optimization stagnated (did not produce new results) for {} seconds".format(self.getName(), timeStagnated))
return
# We will continue. Check if load balancing needs to be done
if now - self.lastBalanceTime >= LOAD_BALANCE_INTERVAL:
self.lastBalanceTime = now
if numActiveRunners > self.maxCores:
# not converged yet + too many active; limit some runners
cpuTimes = [(r.currentCpuTime, r) for r in self.runners if r.isActive]
cpuTimes.sort()
# continue first `maxCores` runners, suspend the rest
resumeRunners = cpuTimes[:self.maxCores]
suspendRunners = cpuTimes[self.maxCores:]
for _,j in resumeRunners:
j.suspend(False)
for _,j in suspendRunners:
j.suspend(True)
else:
for r in self.runners:
r.suspend(False)
