Пример #1
0
    def createRunners(self):
        # move the current runners, if any, to the array with old runners
        if self.runners:
            self.oldCpuTimes.append(max(r.currentCpuTime for r in self.runners))
            self.oldRunners.extend(self.runners)
            self.runners = []

        # reset other state
        self.convergenceTime = None
        self.lastOfUpdateTime = time.time()

        self.runnerGeneration += 1

        bestParams = None
        if self.oldRunners and bool(g.getConfig("optimization.restartFromBestValue")):
            # get best params
            bestParams = self.getBestParams()

        for id in range(int(g.getConfig("optimization.runsPerJob"))):
            r = runner.Runner(self, id + 1, self.currentMethod, self.runnerGeneration)
            if not r.prepare(self.workDir, self.copasiFile, bestParams):
                g.log(LOG_ERROR, "{}: failed to create a runner".format(r.getName()))
                return False
            self.runners.append(r)

        # note that this may create more processes than the number of free CPU cores!
        for r in self.runners:
            r.execute()

        return True
Пример #2
0
 def __init__(self, pool, params, maxCores, areParametersChangeable):
     self.pool = pool
     self.params = params
     self.areParametersChangeable = areParametersChangeable
     self.id = pool.strategy.nextJobID
     pool.strategy.nextJobID += 1
     self.methods = copy.copy(g.getConfig("copasi.methods"))
     assert len(self.methods)
     self.fallbackMethods = copy.copy(g.getConfig("copasi.fallbackMethods"))
     self.currentMethod = None
     self.pastMethods = []
     self.isUsingFallback = False
     self.runners = []
     self.oldRunners = []
     self.runnerGeneration = 0
     self.oldCpuTimes = []
     self.convergenceTime = None
     self.convergenceValue = None
     self.copasiFile = None
     self.workDir = None
     self.startTime = time.time()
     self.lastOfUpdateTime = self.startTime
     self.lastBalanceTime = self.startTime - LOAD_BALANCE_INTERVAL
     # XXX: note that the number of free cores can increase during job's
     # lifetime, but the code will not react to that, keeping maxCores constant
     self.maxCores = maxCores
     self.loadBalanceFactor = float(g.getConfig("optimization.runsPerJob")) / maxCores
Пример #3
0
    def createRunners(self):
        # move the current runners, if any, to the array with old runners
        if self.runners:
            self.oldCpuTimes.append(max(r.currentCpuTime for r in self.runners))
            self.oldRunners.extend(self.runners)
            self.runners = []

        # reset other state
        self.convergenceTime = None
        self.lastOfUpdateTime = time.time()

        self.runnerGeneration += 1

        bestParams = None
        if self.oldRunners and bool(g.getConfig("optimization.restartFromBestValue")):
            # get best params
            bestParams = self.getBestParams()

        for id in range(int(g.getConfig("optimization.runsPerJob"))):
            r = runner.Runner(self, id + 1, self.currentMethod, self.runnerGeneration)
            if not r.prepare(self.workDir, self.copasiFile, bestParams):
                g.log(LOG_ERROR, "{}: failed to create a runner".format(r.getName()))
                return False
            self.runners.append(r)

        # note that this may create more processes than the number of free CPU cores!
        for r in self.runners:
            r.execute()

        return True
Пример #4
0
 def __init__(self, pool, params, maxCores, areParametersChangeable):
     self.pool = pool
     self.params = params
     self.areParametersChangeable = areParametersChangeable
     self.id = pool.strategy.nextJobID.next()
     self.methods = copy.copy(g.getConfig("copasi.methods"))
     assert len(self.methods)
     self.fallbackMethods = copy.copy(g.getConfig("copasi.fallbackMethods"))
     self.currentMethod = None
     self.isUsingFallback = False
     self.runners = []
     self.oldRunners = []
     self.runnerGeneration = 0
     self.oldCpuTimes = []
     self.convergenceTime = None
     self.convergenceValue = None
     self.copasiFile = None
     self.workDir = None
     self.startTime = time.time()
     self.lastOfUpdateTime = self.startTime
     self.lastBalanceTime = self.startTime - LOAD_BALANCE_INTERVAL
     # XXX: note that the number of free cores can increase during job's
     # lifetime, but the code will not react to that, keeping maxCores constant
     self.maxCores = maxCores
     self.loadBalanceFactor = float(g.getConfig("optimization.runsPerJob")) / maxCores
Пример #5
0
    def totalOptimizationPotentialReached(self, numParameters):
        if not self.isTOPEnabled():
            return False

        targetFraction = g.getConfig("optimization.targetFractionOfTOP")

        # calculate the target value, looking at both config and at the job with all parameters, if any
        try:
            configTarget = float(g.getConfig("optimization.bestOfValue"))
        except:
            g.log(
                LOG_INFO, "Bad bestOfValue in config: {}".format(
                    g.getConfig("optimization.bestOfValue")))
            return False
        joblist = self.jobsByBestOfValue[-1]
        if joblist:
            calculatedTarget = joblist[0].getBestOfValue()
        else:
            calculatedTarget = MIN_OF_VALUE
        targetValue = max(configTarget, calculatedTarget)
        if targetValue == MIN_OF_VALUE:
            g.log(
                LOG_DEBUG,
                "TOP: no target value: {} {}".format(configTarget,
                                                     calculatedTarget))
            return False

        achievedValue = MIN_OF_VALUE
        for joblist in self.jobsByBestOfValue[:numParameters + 1]:
            if joblist:
                achievedValue = max(achievedValue, joblist[0].getBestOfValue())

        isReached = False
        if self.topBaseline > targetValue:
            isReached = True
            requiredValue = targetValue
        else:
            requiredValue = (targetValue - self.topBaseline
                             ) * targetFraction + self.topBaseline
            isReached = achievedValue >= requiredValue

        g.log(
            LOG_DEBUG,
            "TOP: {} parameters, {} achieved, {} required, {} target, {} configTarget, {} calculatedTarget"
            .format(numParameters, achievedValue, requiredValue, targetValue,
                    configTarget, calculatedTarget))

        if isReached:
            g.log(
                LOG_INFO,
                "Terminating optimization at {} parameters: good-enough-value criteria reached (required {})"
                .format(numParameters, requiredValue))
            return True
        return False
Пример #6
0
    def dumpResults(self, totalLimit=0, perParamLimit=0):
        if g.workDir is None:
            return None
        filename = g.getConfig("output.filename")
        # do not allow put the results in other directories because of security reasons
        if filename != os.path.basename(filename):
            g.log(
                LOG_INFO,
                "output file name should not include path, ignoring all but the last element in it"
            )
            filename = os.path.basename(filename)
        (name, ext) = os.path.splitext(filename)
        if not ext:
            ext = ".csv"  # default
        filename = os.path.join(g.workDir,
                                "{}-{}{}".format(name, "-", g.taskName, ext))

        with self.jobLock:
            if len(self.finishedJobs) <= self.lastNumJobsDumped:
                # all finished jobs already were saved, nothing to do
                return filename
            self.lastNumJobsDumped = len(self.finishedJobs)

        allJobsByBestOfValue = []
        if perParamLimit == 0:
            numberOfBestCombinations = int(
                g.getConfig("output.numberOfBestCombinations"))
        else:
            numberOfBestCombinations = perParamLimit
        for joblist in self.jobsByBestOfValue:
            if numberOfBestCombinations:
                lst = joblist[:numberOfBestCombinations]
            else:
                lst = joblist
            for job in lst:
                allJobsByBestOfValue.append(job)
        allJobsByBestOfValue.sort(key=lambda x: x.getBestOfValue(),
                                  reverse=True)

        allParams = self.copasiConfig["params"]

        cnt = 0
        with open(filename, "w") as f:
            self.dumpCsvFileHeader(f)
            for job in allJobsByBestOfValue:
                job.dumpResults(f, allParams)
                cnt += 1
                if totalLimit and cnt >= totalLimit:
                    break

        g.log(
            LOG_INFO, '<spacescanner>: results of finished jobs saved in "' +
            filename + '"')
        return filename
Пример #7
0
 def __init__(self, strategy, parameterSets, areParametersChangeable):
     self.strategy = strategy
     self.parameterSets = parameterSets
     self.areParametersChangeable = areParametersChangeable
     self.currentParametersIndex = 0
     self.jobLock = threading.Lock()
     self.activeJobs = []
     self.numUsableCores = max(1, int(g.getConfig("optimization.maxConcurrentRuns")))
     self.numRunnersPerJob = max(1, int(g.getConfig("optimization.runsPerJob")))
     self.maxNumParallelJobs = int(math.ceil(float(self.numUsableCores) / self.numRunnersPerJob))
     self.bestOfValue = MIN_OF_VALUE
     self.bestParams = []
Пример #8
0
 def __init__(self, strategy, parameterSets, areParametersChangeable):
     self.strategy = strategy
     self.parameterSets = parameterSets
     self.areParametersChangeable = areParametersChangeable
     self.currentParametersIndex = 0
     self.jobLock = threading.Lock()
     self.activeJobs = []
     self.numUsableCores = max(1, int(g.getConfig("optimization.maxConcurrentRuns")))
     self.numRunnersPerJob = max(1, int(g.getConfig("optimization.runsPerJob")))
     self.maxNumParallelJobs = int(math.ceil(float(self.numUsableCores) / self.numRunnersPerJob))
     self.bestOfValue = MIN_OF_VALUE
     self.bestParams = []
Пример #9
0
 def isTOPEnabled(self):
     targetFraction = g.getConfig("optimization.targetFractionOfTOP")
     if targetFraction is None or targetFraction == 0.0:
         # TOP is disabled
         return False
     # TOP is enabled
     return True
Пример #10
0
 def isTOPEnabled(self):
     optimality = g.getConfig("optimization.optimalityRelativeError")
     if optimality is None or optimality == 0.0:
         # TOP is disabled
         return False
     # TOP is enabled
     return True
Пример #11
0
def executeCopasi(runner):
    # a pseudo-loop for simpler error handling
    while True:
        if bool(g.getConfig("webTestMode")):
            if runner.job.id < 4:
                time.sleep(1.0)
                runner.ofValue = random.random() * 100
                runner.isActive = False
            else:
                # wait for the strategy to quit
                while not runner.job.pool.strategy.doQuitFlag:
                    time.sleep(1.0)
                    runner.ofValue += random.random()
                runner.isActive = False
            break

        g.log(LOG_DEBUG, "executing " + " ".join(runner.process.args))
        runner.isError = runner.process.run()

        # check report in order to update OF value (even if nonzero return value)
        with reportLock:
            runner.checkReport(hasTerminated=True, now=time.time())

        # exit the loop without an error
        break

    # check termination conditions at the global run level
    g.log(LOG_DEBUG, "{}: terminated".format(runner.getName()))
    runner.isActive = False
    # disable this; do it from polling in the main thread instead
    #runner.job.checkIfHasTerminated()
    # overwrite the .cps file with best parameter values
    runner.cleanup()
Пример #12
0
def executeCopasi(runner):
    # a pseudo-loop for simpler error handling
    while True:
        if bool(g.getConfig("webTestMode")):
            if runner.job.id < 4:
                time.sleep(1.0)
                runner.ofValue = random.random() * 100
                runner.isActive = False
            else:
                # wait for the strategy to quit
                while not runner.job.pool.strategy.doQuitFlag:
                    time.sleep(1.0)
                    runner.ofValue += random.random()
                runner.isActive = False
            break

        g.log(LOG_DEBUG, "executing " + " ".join(runner.process.args))
        runner.isError = runner.process.run()

        # check report in order to update OF value (even if nonzero return value)
        with reportLock:
            runner.checkReport(hasTerminated=True, now=time.time())

        # exit the loop without an error
        break

    # check termination conditions at the global run level
    g.log(LOG_DEBUG, "{}: terminated".format(runner.getName()))
    runner.isActive = False
    # disable this; do it from polling in the main thread instead
    # runner.job.checkIfHasTerminated()
    # overwrite the .cps file with best parameter values
    runner.cleanup()
Пример #13
0
 def loadCopasiFile(self):
     copasiFile = copasifile.CopasiFile()
     filename = g.getConfig("copasi.modelFile")
     filename = filename.replace("@SELF@", SELF_PATH)
     g.log(LOG_INFO, "<spacescanner>: opening COPASI model file {}".format(filename))
     if not copasiFile.read(filename):
         return None
     return copasiFile
Пример #14
0
    def dumpResults(self, totalLimit = 0, perParamLimit = 0):
        if g.workDir is None:
            return None
        filename = g.getConfig("output.filename")
        # do not allow put the results in other directories because of security reasons
        if filename != os.path.basename(filename):
            g.log(LOG_INFO, "output file name should not include path, ignoring all but the last element in it")
            filename = os.path.basename(filename)
        (name, ext) = os.path.splitext(filename)
        if not ext:
            ext = ".csv" # default
        filename = os.path.join(g.workDir, "{}-{}{}".format(name, "-", g.taskName, ext))

        with self.jobLock:
            if len(self.finishedJobs) <= self.lastNumJobsDumped:
                # all finished jobs already were saved, nothing to do
                return filename
            self.lastNumJobsDumped = len(self.finishedJobs)

        allJobsByBestOfValue = []
        if perParamLimit == 0:
            numberOfBestCombinations = int(g.getConfig("output.numberOfBestCombinations"))
        else:
            numberOfBestCombinations = perParamLimit
        for joblist in self.jobsByBestOfValue:
            if numberOfBestCombinations:
                lst = joblist[:numberOfBestCombinations]
            else:
                lst = joblist
            for job in lst:
                allJobsByBestOfValue.append(job)
        allJobsByBestOfValue.sort(key=lambda x: x.getBestOfValue(), reverse=True)

        allParams = self.copasiConfig["params"]

        cnt = 0
        with open(filename, "w") as f:
            self.dumpCsvFileHeader(f)
            for job in allJobsByBestOfValue:
                job.dumpResults(f, allParams)
                cnt += 1
                if totalLimit and cnt >= totalLimit:
                    break

        g.log(LOG_INFO, '<spacescanner>: results of finished jobs saved in "' + filename + '"')
        return filename
Пример #15
0
 def loadCopasiFile(self):
     copasiFile = copasifile.CopasiFile()
     filename = g.getConfig("copasi.modelFile")
     filename = filename.replace("@SELF@", SELF_PATH)
     g.log(LOG_INFO,
           "<spacescanner>: opening COPASI model file {}".format(filename))
     if not copasiFile.read(filename):
         return None
     return copasiFile
Пример #16
0
    def hasConsensus(self):
        epsilonAbs = float(g.getConfig("optimization.consensusAbsoluteError"))
        epsilonRel = float(g.getConfig("optimization.consensusRelativeError"))

        if self.convergenceTime is None:
            minV = min([r.ofValue for r in self.runners])
        else:
            minV = self.convergenceValue
        # if this is not the first method, also should use max from the previous
        maxV = self.getBestOfValue()

        # returns true if either the absolute difference OR relative difference are small
        if floatEqual(minV, maxV, epsilonAbs):
            return True
        # XXX: avoid division by zero; this means relative convergence will always fail on 0.0
        if math.isinf(minV) or math.isinf(maxV) or maxV == 0.0:
            return False
        return abs(1.0 - minV / maxV) < epsilonRel
Пример #17
0
 def ioGetActiveJobs(self, qs):
     response = []
     with self.jobLock:
         if bool(g.getConfig("webTestMode")):
             for id in self.finishedJobs:
                 if id < 4:
                     response.append(self.finishedJobs[id].getStatsFull())
         if self.activeJobPool:
             with self.activeJobPool.jobLock:
                 for job in self.activeJobPool.activeJobs:
                     response.append(job.getStatsFull())
     return response
Пример #18
0
    def execute(self, workDir, copasiFile):
        self.workDir = workDir
        self.copasiFile = copasiFile

        if bool(g.getConfig("optimization.randomizeMethodSelection")):
            self.currentMethod = random.choice(self.methods)
        else:
            self.currentMethod = self.methods[0]

        g.log(LOG_INFO, "starting " + self.getFullName())

        return self.createRunners()
Пример #19
0
    def execute(self, workDir, copasiFile):
        self.workDir = workDir
        self.copasiFile = copasiFile

        if bool(g.getConfig("optimization.randomizeMethodSelection")):
            self.currentMethod = random.choice(self.methods)
        else:
            self.currentMethod = self.methods[0]

        g.log(LOG_INFO, "starting " + self.getFullName())

        return self.createRunners()
Пример #20
0
    def totalOptimizationPotentialReached(self, numParameters):
        if not self.isTOPEnabled():
            return False

        optimality = g.getConfig("optimization.optimalityRelativeError")

        # calculate the target value, looking at both config and at the job with all parameters, if any
        try:
            configTarget = float(g.getConfig("optimization.bestOfValue"))
        except:
            g.log(LOG_INFO, "Bad bestOfValue in config: {}".format(g.getConfig("optimization.bestOfValue")))
            return False
        joblist = self.jobsByBestOfValue[-1]
        if joblist:
            calculatedTarget = joblist[0].getBestOfValue()
        else:
            calculatedTarget = MIN_OF_VALUE
        targetValue = max(configTarget, calculatedTarget)
        if targetValue == MIN_OF_VALUE:
            return False

        achievedValue = MIN_OF_VALUE
        for joblist in self.jobsByBestOfValue[:numParameters + 1]:
            if joblist:
                achievedValue = max(achievedValue, joblist[0].getBestOfValue())

        proportion = 1.0 - float(optimality)
        isReached = False
        if self.topBaseline > targetValue:
            isReached = True
            requiredValue = targetValue
        else:
            requiredValue = (targetValue - self.topBaseline) * proportion + self.topBaseline
            isReached = achievedValue >= requiredValue

        if isReached:
            g.log(LOG_INFO, "terminating optimization at {} parameters: good-enough-value criteria reached (required {})".format(numParameters, requiredValue))
            return True
        return False
Пример #21
0
    def getAllStats(self):
        if bool(g.getConfig("webTestMode")):
            r = 0.0
            t = 0.0
            random.seed(0)
            result = []
            for i in range(20):
                r += random.random()
                t += random.random()
                si = StatsItem("{} {} 1 ( 74.24 ) -1.81914".format(r, t))
                result.append(si)
            return result

        return self.getAllStatsForGeneration(self.generation)
Пример #22
0
    def getAllStats(self):
        if bool(g.getConfig("webTestMode")):
            r = 0.0
            t = 0.0
            random.seed(0)
            result = []
            for i in range(20):
                r += random.random()
                t += random.random()
                si = StatsItem("{} {} 1 ( 74.24 ) -1.81914".format(r, t))
                result.append(si)
            return result

        return self.getAllStatsForGeneration(self.generation)
Пример #23
0
    def hasConsensus(self):
        epsilonAbs = float(g.getConfig("optimization.consensusAbsoluteError"))
        epsilonRel = float(g.getConfig("optimization.consensusCorridor"))

        if self.convergenceTime is None:
            minV = min([r.ofValue for r in self.runners])
        else:
            minV = self.convergenceValue
        # if this is not the first method, also should use max from the previous
        maxV = self.getBestOfValue()

        # return False if exited without a result
        if math.isinf(minV) or math.isinf(maxV):
            return False

        # returns true if either the absolute difference OR relative difference are small
        if floatEqual(minV, maxV, epsilonAbs):
            return True

        # XXX: avoid division by zero; this means relative convergence will always fail on 0.0
        if maxV == 0.0:
            return False

        return abs(1.0 - minV / maxV) < epsilonRel
Пример #24
0
def start(configFileName):
    if not g.prepare(configFileName):
        return

    # read COPASI model file etc.
    strategyManager = strategy.StrategyManager()
    if not strategyManager.prepare(isDummy=False):
        return

    # start the web server
    if bool(g.getConfig("web.enable")):
        process.createBackgroundThread(executeWebserver, strategyManager)

    # start the selected parameter sweep strategy
    strategyManager.execute()
Пример #25
0
def executeWebserver(strategyManager):
    try:
        port = int(g.getConfig("web.port"))
        g.log(LOG_INFO, "<spacescanner>: starting webserver, port: " + str(port))
        server = webserver.InterruptibleHTTPServer(('', port), webserver.HttpServerHandler)
        server.strategyManager = strategyManager
        # report ok and enter the main loop
        g.log(LOG_DEBUG, "<spacescanner>: webserver started, listening to port {}".format(port))
        server.serve_forever()
    except Exception as e:
        g.log(LOG_ERROR, "<spacescanner>: exception occurred in webserver:")
        g.log(LOG_ERROR, str(e))
        g.log(LOG_ERROR, traceback.format_exc())
        sys.exit(1)
    sys.exit(0)
Пример #26
0
def start(configFileName):
    if not g.prepare(configFileName):
        return

    # read COPASI model file etc.
    strategyManager = strategy.StrategyManager()
    if not strategyManager.prepare(isDummy = False):
        return

    # start the web server
    if bool(g.getConfig("web.enable")):
        process.createBackgroundThread(executeWebserver, strategyManager)

    # start the selected parameter sweep strategy
    strategyManager.execute()
Пример #27
0
    def serializeOptimizationTask(self, reportFilename, outf, parameters, methodNames, startParamValues, areParametersChangeable):
        # Note: 'scheduled' is always set to true, as is 'update model' to save the final parameter values in the .cps file.
        outf.write('  <Task key="{}" name="Optimization" type="optimization" scheduled="true" updateModel="true">\n'.format(self.optimizationTask.get("key")))

        # 1. Fix report target file name
        # <Report reference="Report_10" target="./report.log" append="1"/>
        report = self.optimizationTask.find('copasi:Report', COPASI_NS)
        if report is not None:
            report.set("target", reportFilename)
            report.set("reference", "optimization_report")
            outf.write('    ' + str(ElementTree.tostring(report)))

        # 2. Include only required parameters
        problem = self.optimizationTask.find('copasi:Problem', COPASI_NS)
        outf.write('    <Problem>\n')
        for elem in problem.iterfind("*", COPASI_NS):
            if "Problem" in elem.tag: continue
            if "ParameterGroup" not in elem.tag or elem.get("name").lower() != "optimizationitemlist":
                if "Parameter" in elem.tag and elem.get("name").lower() == "randomize start values":
                    # never randomize them
                    outf.write('    <Parameter name="Randomize Start Values" type="bool" value="0"/>\n')
                else:
                    outf.write('    ' + str(ElementTree.tostring(elem)))
                continue

        #print("parameters in the file:", self.paramDict.keys())
        outf.write(' <ParameterGroup name="OptimizationItemList">\n')
        for p in parameters:
            if p in self.paramDict:
                self.writeParam(outf, p, startParamValues, areParametersChangeable)
        outf.write(' </ParameterGroup>\n')
        outf.write(' </Problem>\n')

        # 3. Include only required methods
        #print("methods in the file:", self.methodDict.keys())
        for m in methodNames:
            methodFromFile = self.methodDict.get(m.lower())
            if bool(g.getConfig("methodParametersFromFile")) and methodFromFile is not None:
                outf.write('        ' + str(ElementTree.tostring(methodFromFile)))
            else:
                predefinedMethod = PREDEFINED_METHODS.get(m)
                if predefinedMethod is None:
                    g.log(LOG_ERROR, "Unknown or unsupported optimization method {}".format(m))
                else:
                    outf.write('        ' + predefinedMethod)

        # finish off
        outf.write('\n  </Task>\n')
Пример #28
0
 def ioGetActiveJobs(self, qs):
     response = {
         "baseline": jsonFixInfinity(self.topBaseline, 0.0),
         "stats": []
     }
     with self.jobLock:
         if bool(g.getConfig("webTestMode")):
             for id in self.finishedJobs:
                 if id < 4:
                     response["stats"].append(
                         self.finishedJobs[id].getStatsFull())
         if self.activeJobPool:
             with self.activeJobPool.jobLock:
                 for job in self.activeJobPool.activeJobs:
                     if job.areParametersChangeable:
                         response["stats"].append(job.getStatsFull())
     return response
Пример #29
0
def executeWebserver(strategyManager):
    try:
        port = int(g.getConfig("web.port"))
        g.log(LOG_INFO,
              "<spacescanner>: starting webserver, port: " + str(port))
        server = webserver.InterruptibleHTTPServer(('', port),
                                                   webserver.HttpServerHandler)
        server.strategyManager = strategyManager
        # report ok and enter the main loop
        g.log(
            LOG_DEBUG,
            "<spacescanner>: webserver started, listening to port {}".format(
                port))
        server.serve_forever()
    except Exception as e:
        g.log(LOG_ERROR, "<spacescanner>: exception occurred in webserver:")
        g.log(LOG_ERROR, str(e))
        g.log(LOG_ERROR, traceback.format_exc())
        sys.exit(1)
    sys.exit(0)
Пример #30
0
def main():
    if len(sys.argv) > 1 and sys.argv[1] == "web":
        # web-only mode; load the last saved web config, if present
        g.loadConfig(os.path.join(SELF_PATH, "tmpweb", "config.json"), isQuiet = True)
        # create an empty strategy and wait for input commands
        strategyManager = strategy.StrategyManager()
        strategyManager.prepare(isDummy = True)
        # start the web server
        process.createBackgroundThread(executeWebserver, strategyManager)
        wait()
    else:
        # normal mode
        if not start(sys.argv[1] if len(sys.argv) > 1 else None):
            return -1

        # if "hang" mode is configured, do not quit until Ctrl+C is pressed
        if bool(g.getConfig("hangMode")):
            wait()

    return 0
Пример #31
0
def main():
    if len(sys.argv) > 1 and sys.argv[1] == "web":
        # web-only mode; load the last saved web config, if present
        g.loadConfig(os.path.join(SELF_PATH, "tmpweb", "config.json"),
                     isQuiet=True)
        # create an empty strategy and wait for input commands
        strategyManager = strategy.StrategyManager()
        strategyManager.prepare(isDummy=True)
        # start the web server
        process.createBackgroundThread(executeWebserver, strategyManager)
        wait()
    else:
        # normal mode
        if not start(sys.argv[1] if len(sys.argv) > 1 else None):
            return -1

        # if "hang" mode is configured, do not quit until Ctrl+C is pressed
        if bool(g.getConfig("hangMode")):
            wait()

    return 0
Пример #32
0
    def execute(self):
        parameterSelections = []
        if g.getConfig("restartOnFile"):
            # Guess which parameters have not been optimized yet based on the .csv result file
            filename = g.getConfig("restartOnFile").replace("@SELF@", SELF_PATH)
            parameterSets = getNonconvergedResults(filename)
            for ps in parameterSets:
                spec = {"type" : "explicit", "parameters" : ps}
                x = ParamSelection.create(spec, self)
                if x not in parameterSelections:
                    parameterSelections.append(x)

        elif g.getConfig("parameters"):

            # Deal with TOP, if required
            if self.isTOPEnabled():
                # add the "zero" at the start
                g.log(LOG_INFO, "optimizing for zero parameters initially to find the baseline")
                spec = {"type" : "zero"}
                parameterSelections.append(ParamSelection.create(spec, self))

            # The take the other sets from the file
            for spec in g.getConfig("parameters"):
                x = ParamSelection.create(spec, self)
                if x is None:
                    g.log(LOG_ERROR, "invalid parameter specification: {}".format(ENC.encode(spec)))
                    continue
                if x not in parameterSelections:
                    parameterSelections.append(x)

        else:
            # add the default optimization target: all parameters
            g.log(LOG_INFO, "optimizing only for all parameters")
            spec = {"type" : "full-set"}
            parameterSelections.append(ParamSelection.create(spec, self))

        numCombinations = 0
        for sel in parameterSelections:
            numCombinations += sel.getNumCombinations()

        g.log(LOG_INFO, "total {} parameter combination(s) to try out, parameters: {}".format(numCombinations, " ".join(self.copasiConfig["params"])))
        g.log(LOG_INFO, "methods enabled: '" + "' '".join(g.getConfig("copasi.methods")) + "'")

        parameterSelections.sort(key = lambda x: x.getSortOrder())
        for sel in parameterSelections:
            areParametersChangeable = sel.type != PARAM_SEL_ZERO
            for params in sel.getParameterSets():
                g.log(LOG_DEBUG, "made a new pool of {} jobs".format(len(params)))
                pool = jobpool.JobPool(self, params, areParametersChangeable)
                with self.jobLock:
                    self.activeJobPool = pool

                pool.start()
                while True:
                    time.sleep(1.0)
                    if self.doQuitFlag:
                        return True

                    try:
                        pool.refresh()
                    except Exception as e:
                        g.log(LOG_INFO, "Exception while refreshing active joob pool status, terminating the pool: {}".format(e))
                        self.finishActivePool()
                        break

                    if pool.isDepleted():
                        self.finishActivePool()
                        break               

        return True
Пример #33
0
    def checkReport(self, hasTerminated, now):
        assert self.isActive

        if not hasTerminated:
            if self.lastReportCheckTime is not None and now - self.lastReportCheckTime < MIN_REPORT_CHECK_INTERVAL:
                # too soon, skip
                return True
            if now - self.startTime < 3.0:
                # too soon after start (copasi may not be launched yet), return
                return True

        self.lastReportCheckTime = now
        oldOfValue = self.ofValue

        try:
            st = os.stat(self.reportFilename)
            if self.lastReportModificationTime is None or st.st_mtime > self.lastReportModificationTime:
                self.lastReportModificationTime = st.st_mtime
                with open(self.reportFilename, "r") as f:
                    self.stats.isValid = False
                    inValues = False
                    for line in f:
                        if startsWith(line, "CPU time"):
                            inValues = True
                            continue
                        if not inValues:
                            continue

                        if startsWith(line, "Optimization Result"):
                            break

                        si = StatsItem(line)
                        if si.isValid:
                            self.stats = si

                if self.stats.isValid:
                    self.ofValue = self.getLastStats().ofValue
                    if oldOfValue != self.ofValue:
                        g.log(LOG_INFO, "{}: new OF value {}".format(self.getName(), self.ofValue))

                    # Check for CPU time end condition using the report file
                    self.currentCpuTime = self.getLastStats().cpuTime
            else:
                # no report file update; check for CPU time end condition using OS measurements
                if not hasTerminated:
                    try:
                        self.currentCpuTime = self.process.getCpuTime()
                    except psutil.NoSuchProcess:
                        pass  # has already quit

        except OSError as e:
            g.log(LOG_ERROR, "accessing report file {} failed: {}".format(self.reportFilename, os.strerror(e.errno)))
        except IOError as e:
            g.log(LOG_ERROR, "parsing report file {} failed: {}".format(self.reportFilename, os.strerror(e.errno)))

        if not hasTerminated and not self.terminationReason:
            g.log(LOG_DEBUG, "checked {}, CPU time: {}".format(self.getName(), self.currentCpuTime))
            # XXX: hardcoded "slow" method names
            if self.methodName not in ["ScatterSearch", "SimulatedAnnealing"]:
                # XXX: reuse consensus time for this
                maxTimeWithNoValue = float(g.getConfig("optimization.consensusMinDurationSec")) + 10.0
                if self.job.timeDiffExceeded(now - self.startTime, maxTimeWithNoValue) and (
                    not self.stats.isValid or self.ofValue == MIN_OF_VALUE
                ):
                    g.log(
                        LOG_DEBUG,
                        "terminating {}: no value found in CPU time: {}".format(self.getName(), self.currentCpuTime),
                    )
                    self.terminationReason = TERMINATION_REASON_CPU_TIME_LIMIT

        if oldOfValue != self.ofValue:
            # the OF value was updated
            return True

        return False  # the OF value was not updated
Пример #34
0
    def execute(self):
        parameterSelections = []

        # always add the zero'th job at start, needed to show baseline in graphs and for TOP
        g.log(LOG_INFO,
              "optimizing for zero parameters initially to find the baseline")
        spec = {"type": "zero"}
        parameterSelections.append(ParamSelection.create(spec, self))

        if self.isTOPEnabled():
            # add all parameters: will define the target value
            spec = {"type": "full-set"}
            parameterSelections.append(ParamSelection.create(spec, self))

        if g.getConfig("restartOnFile"):
            # Guess which parameters have not been optimized yet based on the .csv result file
            filename = g.getConfig("restartOnFile").replace(
                "@SELF@", SELF_PATH)
            parameterSets = getNonconvergedResults(filename)
            for ps in parameterSets:
                spec = {"type": "explicit", "parameters": ps}
                x = ParamSelection.create(spec, self)
                if x not in parameterSelections:
                    parameterSelections.append(x)

        elif g.getConfig("parameters"):
            # Take the paramter sets from the file
            for spec in g.getConfig("parameters"):
                x = ParamSelection.create(spec, self)
                if x is None:
                    g.log(
                        LOG_ERROR,
                        "invalid parameter specification: {}".format(
                            ENC.encode(spec)))
                    continue
                if x not in parameterSelections:
                    parameterSelections.append(x)

        else:
            # add the default optimization target: all parameters
            g.log(LOG_INFO, "optimizing only for all parameters")
            spec = {"type": "full-set"}
            parameterSelections.append(ParamSelection.create(spec, self))

        hashes = set()
        for sel in parameterSelections:
            hashes = hashes.union(sel.getAllJobHashes())
        self.totalNumJobs = len(hashes) - 1

        g.log(
            LOG_INFO,
            "total {} parameter combination(s) to try out, parameters: {}".
            format(self.totalNumJobs, " ".join(self.copasiConfig["params"])))
        g.log(
            LOG_INFO, "methods enabled: '" +
            "' '".join(g.getConfig("copasi.methods")) + "'")

        parameterSelections.sort(key=lambda x: x.getSortOrder())
        for sel in parameterSelections:
            g.log(LOG_DEBUG,
                  "processing parameter selection of type {}".format(sel.type))
            for params in sel.getParameterSets():
                g.log(LOG_DEBUG,
                      "made a new pool of {} jobs".format(len(params)))
                pool = jobpool.JobPool(self, params,
                                       sel.areParametersChangeable())
                with self.jobLock:
                    self.activeJobPool = pool

                pool.start()
                while True:
                    time.sleep(1.0)
                    if self.doQuitFlag:
                        return True

                    try:
                        pool.refresh()
                    except Exception as e:
                        g.log(
                            LOG_INFO,
                            "Exception while refreshing active joob pool status, terminating the pool: {}"
                            .format(e))
                        self.finishActivePool()
                        break

                    if pool.isDepleted():
                        self.finishActivePool()
                        break

        return True
Пример #35
0
    def checkReports(self):
        # if no runners are active, quit
        if not any([r.isActive for r in self.runners]):
            if self.convergenceTime is not None:
                minAbsoluteTime = float(g.getConfig("optimization.consensusMinDurationSec"))
                # XXX: do not check the relative time here
                if self.hasConsensus() and self.timeDiffExceeded(time.time() - self.convergenceTime, minAbsoluteTime):
                    # count COPASI termination as consensus in this case
                    # XXX: note that this does *not* overwrite "time limit exceeded" exit code!
                    for r in self.runners:
                        if r.terminationReason == TERMINATION_REASON_COPASI_FINISHED:
                            r.terminationReason = TERMINATION_REASON_CONSENSUS
            # switch the methods if required
            self.decideTermination()
            return

        if not all([r.isActive for r in self.runners]):
            # some but not all have quit; quit the others with "stagnation"
            # (not technically true, but the best match from the existing codes)
            for r in self.runners:
                if r.isActive and not r.terminationReason:
                    r.terminationReason = TERMINATION_REASON_STAGNATION
            return

        numActiveRunners = 0
        now = time.time()
        cpuTimeLimit = float(g.getConfig("optimization.timeLimitSec"))
        maxCpuTime = 0
        anyUpdated = False
        with runner.reportLock:
            for r in self.runners:
                if r.isActive:
                    numActiveRunners += 1
                    if r.checkReport(hasTerminated=False, now=now):
                        # the runner updated the OF time
                        self.lastOfUpdateTime = now
                        anyUpdated = True
                    maxCpuTime = max(maxCpuTime, r.currentCpuTime)

        if not self.areParametersChangeable:
            # it is simple; as soon as the first value is read, return
            if anyUpdated:
                for r in self.runners:
                    if r.isActive:
                        r.terminationReason = TERMINATION_REASON_CONSENSUS
                return

        consensusReached = self.hasConsensus()

        if all([r.terminationReason for r in self.runners]):
            return

        # use the old CPU time as a basis
        maxCpuTime += sum(self.oldCpuTimes)
        doKillOnTimeLimit = maxCpuTime >= cpuTimeLimit and not consensusReached
        if doKillOnTimeLimit:
            # kill all jobs immediately
            for r in self.runners:
                if r.isActive:
                    r.terminationReason = TERMINATION_REASON_CPU_TIME_LIMIT
                    g.log(
                        LOG_INFO,
                        "terminating {}: CPU time limit exceeded ({} vs. {})".format(
                            r.getName(), r.currentCpuTime, cpuTimeLimit
                        ),
                    )
            return

        # check if the runs have reached consensus
        if consensusReached:
            if self.convergenceTime is None:
                g.log(LOG_DEBUG, self.getName() + ": reached consensus, waiting for guard time before termination")
                self.convergenceTime = now
                self.convergenceValue = min([r.ofValue for r in self.runners])

            # if the runners have converged for long enough time, quit
            else:
                timeConverged = time.time() - self.convergenceTime
                minAbsoluteTime = float(g.getConfig("optimization.consensusMinDurationSec"))
                minRelativeTime = (time.time() - self.startTime) * float(
                    g.getConfig("optimization.consensusMinProportionalDuration")
                )
                if self.timeDiffExceeded(timeConverged, minAbsoluteTime) and timeConverged > minRelativeTime:
                    g.log(LOG_INFO, "terminating {}: consensus reached".format(self.getName()))
                    for r in self.runners:
                        if r.isActive:
                            r.terminationReason = TERMINATION_REASON_CONSENSUS
                    self.convergenceTime = now
                    return  # do not check other criteria
        else:
            # reset the timer
            self.convergenceTime = None

            # check for stagnation's time limit
            timeStagnated = time.time() - self.lastOfUpdateTime
            maxAbsoluteTime = float(g.getConfig("optimization.stagnationMaxDurationSec"))
            maxRelativeTime = (time.time() - self.startTime) * float(
                g.getConfig("optimization.stagnationMaxProportionalDuration")
            )
            if self.timeDiffExceeded(timeStagnated, maxAbsoluteTime) and timeStagnated > maxRelativeTime:
                # stagnation detected
                for r in self.runners:
                    if r.isActive:
                        r.terminationReason = TERMINATION_REASON_STAGNATION
                    g.log(
                        LOG_INFO,
                        "terminating {}: Optimization stagnated (did not produce new results) for {} seconds".format(
                            self.getName(), timeStagnated
                        ),
                    )
                return

        # We will continue. Check if load balancing needs to be done
        if now - self.lastBalanceTime >= LOAD_BALANCE_INTERVAL:
            self.lastBalanceTime = now
            if numActiveRunners > self.maxCores:
                # not converged yet + too many active; limit some runners
                cpuTimes = [(r.currentCpuTime, r) for r in self.runners if r.isActive]
                cpuTimes.sort()
                # continue first `maxCores` runners, suspend the rest
                resumeRunners = cpuTimes[: self.maxCores]
                suspendRunners = cpuTimes[self.maxCores :]
                for _, j in resumeRunners:
                    j.suspend(False)
                for _, j in suspendRunners:
                    j.suspend(True)
            else:
                for r in self.runners:
                    r.suspend(False)
Пример #36
0
    def decideTermination(self):
        continuableReasons = [TERMINATION_REASON_STAGNATION, TERMINATION_REASON_COPASI_FINISHED]
        if not any([r.terminationReason in continuableReasons for r in self.runners]):
            # all terminated with consensus, because asked by the user, or with time limit
            self.pool.finishJob(self)
            return

        # So we have at least one termination with either:
        #  a) the stagnation limit was exceeded, or
        #  b) Copasi stopped without consensus.
        # Actions now:
        #  1) if no solution found: use a fallback method;
        #  2) else switch to the next method;
        #  3) if no more methods are available, quit.

        if self.currentMethod in self.fallbackMethods:
            # remove the already-used method to avoid infinite looping between methods
            self.fallbackMethods.remove(self.currentMethod)

        assert self.currentMethod in self.methods
        self.methods.remove(self.currentMethod)

        anyNotFound = any([math.isinf(r.ofValue) for r in self.runners])
        if anyNotFound or self.isUsingFallback:
            if len(self.fallbackMethods) == 0:
                if anyNotFound:
                    g.log(LOG_INFO, "terminating {}: failed to evaluate the objective function".format(self.getName()))
                else:
                    g.log(
                        LOG_INFO,
                        "terminating {}: all fallback methods exhausted without reaching consensus".format(
                            self.getName()
                        ),
                    )
                self.pool.finishJob(self)
                return

            if bool(g.getConfig("optimization.randomizeMethodSelection")):
                self.currentMethod = random.choice(self.fallbackMethods)
            else:
                self.currentMethod = self.fallbackMethods[0]
            # make sure the fallback methods are also in methods
            if self.currentMethod not in self.methods:
                self.methods.append(self.currentMethod)
            g.log(LOG_INFO, "switching {} to a fallback method {}".format(self.getName(), self.currentMethod))
            self.isUsingFallback = True
            # switch to the fallback method
            if not self.createRunners():
                self.pool.finishJob(self)
            return

        if len(self.methods) == 0:
            g.log(LOG_INFO, "terminating {}: all methods exhausted without reaching consensus".format(self.getName()))
            self.pool.finishJob(self)
            return

        # go for the next method
        if bool(g.getConfig("optimization.randomizeMethodSelection")):
            self.currentMethod = random.choice(self.methods)
        else:
            self.currentMethod = self.methods[0]

        g.log(LOG_INFO, "switching {} to the next method {}".format(self.getName(), self.currentMethod))
        if not self.createRunners():
            self.pool.finishJob(self)
            return
Пример #37
0
    def serializeOptimizationTask(self, reportFilename, outf, parameters,
                                  methodNames, startParamValues,
                                  areParametersChangeable):
        # Note: 'scheduled' is always set to true, as is 'update model' to save the final parameter values in the .cps file.
        outf.write(
            '  <Task key="{}" name="Optimization" type="optimization" scheduled="true" updateModel="true">\n'
            .format(self.optimizationTask.get("key")))

        # 1. Fix report target file name
        # <Report reference="Report_10" target="./report.log" append="1"/>
        report = self.optimizationTask.find('copasi:Report', COPASI_NS)
        if report is not None:
            report.set("target", reportFilename)
            report.set("reference", "optimization_report")
            outf.write('    ' + str(ElementTree.tostring(report)))

        # 2. Include only required parameters
        problem = self.optimizationTask.find('copasi:Problem', COPASI_NS)
        outf.write('    <Problem>\n')
        for elem in problem.iterfind("*", COPASI_NS):
            if "Problem" in elem.tag: continue
            if "ParameterGroup" not in elem.tag or elem.get(
                    "name").lower() != "optimizationitemlist":
                if "Parameter" in elem.tag and elem.get(
                        "name").lower() == "randomize start values":
                    # never randomize them
                    outf.write(
                        '    <Parameter name="Randomize Start Values" type="bool" value="0"/>\n'
                    )
                else:
                    outf.write('    ' + str(ElementTree.tostring(elem)))
                continue

        #print("parameters in the file:", self.paramDict.keys())
        outf.write(' <ParameterGroup name="OptimizationItemList">\n')
        for p in parameters:
            if p in self.paramDict:
                self.writeParam(outf, p, startParamValues,
                                areParametersChangeable)
        outf.write(' </ParameterGroup>\n')
        outf.write(' </Problem>\n')

        # 3. Include only required methods
        #print("methods in the file:", self.methodDict.keys())
        for m in methodNames:
            methodFromFile = self.methodDict.get(m.lower())
            if bool(g.getConfig("methodParametersFromFile")
                    ) and methodFromFile is not None:
                outf.write('        ' +
                           str(ElementTree.tostring(methodFromFile)))
            else:
                predefinedMethod = PREDEFINED_METHODS.get(m)
                if predefinedMethod is None:
                    g.log(
                        LOG_ERROR,
                        "Unknown or unsupported optimization method {}".format(
                            m))
                else:
                    outf.write('        ' + predefinedMethod)

        # finish off
        outf.write('\n  </Task>\n')
Пример #38
0
    def checkReports(self):
        # if no runners are active, quit
        if not any([r.isActive for r in self.runners]):
            if self.convergenceTime is not None:
                minAbsoluteTime = float(g.getConfig("optimization.consensusDelaySec"))
                # XXX: do not check the relative time here
                if self.hasConsensus() and self.timeDiffExceeded(time.time() - self.convergenceTime, minAbsoluteTime):
                    # count COPASI termination as consensus in this case
                    # XXX: note that this does *not* overwrite "time limit exceeded" exit code!
                    for r in self.runners:
                        if r.terminationReason == TERMINATION_REASON_COPASI_FINISHED:
                            r.terminationReason = TERMINATION_REASON_CONSENSUS
            # switch the methods if required
            self.decideTermination()
            return

        if not all([r.isActive for r in self.runners]):
            # some but not all have quit; quit the others with "stagnation"
            # (not technically true, but the best match from the existing codes)
            for r in self.runners:
                if r.isActive and not r.terminationReason:
                    r.terminationReason = TERMINATION_REASON_STAGNATION
            return

        numActiveRunners = 0
        now = time.time()
        cpuTimeLimit = float(g.getConfig("optimization.timeLimitSec"))
        maxCpuTime = 0
        anyUpdated = False
        with runner.reportLock:
            for r in self.runners:
                if r.isActive:
                    numActiveRunners += 1
                    if r.checkReport(hasTerminated = False, now = now):
                        # the runner updated the OF time
                        self.lastOfUpdateTime = now
                        anyUpdated = True
                    maxCpuTime = max(maxCpuTime, r.currentCpuTime)

        if not self.areParametersChangeable:
            # it is simple; as soon as the first value is read, return
            if anyUpdated:
                for r in self.runners:
                    if r.isActive:
                        r.terminationReason = TERMINATION_REASON_CONSENSUS
                return

        consensusReached = self.hasConsensus()

        if all([r.terminationReason for r in self.runners]):
            return

        # use the old CPU time as a basis
        maxCpuTime += sum(self.oldCpuTimes)
        doKillOnTimeLimit = maxCpuTime >= cpuTimeLimit and not consensusReached
        if doKillOnTimeLimit:
            # kill all jobs immediately
            for r in self.runners:
                if r.isActive:
                    r.terminationReason = TERMINATION_REASON_CPU_TIME_LIMIT
                    g.log(LOG_INFO, "terminating {}: CPU time limit exceeded ({} vs. {})".format(
                        r.getName(), maxCpuTime, cpuTimeLimit))
            return

        # check if the runs have reached consensus
        if consensusReached:
            if self.convergenceTime is None:
                g.log(LOG_DEBUG, self.getName() + ": reached consensus, waiting for guard time before termination")
                self.convergenceTime = now
                self.convergenceValue = min([r.ofValue for r in self.runners])

            # if the runners have converged for long enough time, quit
            else:
                timeConverged = time.time() - self.convergenceTime
                minAbsoluteTime = float(g.getConfig("optimization.consensusDelaySec"))
                minRelativeTime = (time.time() - self.startTime) * float(g.getConfig("optimization.consensusProportionalDelay"))
                if self.timeDiffExceeded(timeConverged, minAbsoluteTime) and timeConverged > minRelativeTime:
                    g.log(LOG_INFO, "terminating {}: consensus reached".format(self.getName()))
                    for r in self.runners:
                        if r.isActive:
                            r.terminationReason = TERMINATION_REASON_CONSENSUS
                    self.convergenceTime = now
                    return # do not check other criteria
        else:
            # reset the timer
            self.convergenceTime = None

            # check for stagnation's time limit
            timeStagnated = time.time() - self.lastOfUpdateTime
            maxAbsoluteTime = float(g.getConfig("optimization.stagnationDelaySec"))
            maxRelativeTime = (time.time() - self.startTime) * float(g.getConfig("optimization.stagnationProportionalDelay"))

            # XXX: specialcase for the non-parameters job: quit it quite quickly (in 10 seconds for each method)
            if not self.areParametersChangeable:
                maxAbsoluteTime = 10.0
                maxRelativeTime = 0.0

            if self.timeDiffExceeded(timeStagnated, maxAbsoluteTime) and timeStagnated > maxRelativeTime:
                # stagnation detected
                for r in self.runners:
                    if r.isActive:
                        r.terminationReason = TERMINATION_REASON_STAGNATION
                    g.log(LOG_INFO, "terminating {}: Optimization stagnated (did not produce new results) for {} seconds".format(self.getName(), timeStagnated))
                return

        # We will continue. Check if load balancing needs to be done
        if now - self.lastBalanceTime >= LOAD_BALANCE_INTERVAL:
            self.lastBalanceTime = now
            if numActiveRunners > self.maxCores:
                # not converged yet + too many active; limit some runners
                cpuTimes = [(r.currentCpuTime, r) for r in self.runners if r.isActive]
                cpuTimes.sort()
                # continue first `maxCores` runners, suspend the rest
                resumeRunners = cpuTimes[:self.maxCores]
                suspendRunners = cpuTimes[self.maxCores:]
                for _,j in resumeRunners:
                    j.suspend(False)
                for _,j in suspendRunners:
                    j.suspend(True)
            else:
                for r in self.runners:
                    r.suspend(False)
Пример #39
0
    def decideTermination(self):
        continuableReasons = [TERMINATION_REASON_STAGNATION,
                              TERMINATION_REASON_COPASI_FINISHED]
        if not any([r.terminationReason in continuableReasons for r in self.runners]):
            # all terminated with consensus, because asked by the user, or with time limit
            self.pool.finishJob(self)
            return

        # So we have at least one termination with either:
        #  a) the stagnation limit was exceeded, or
        #  b) Copasi stopped without consensus.
        # Actions now:
        #  1) if no solution found: use a fallback method;
        #  2) else switch to the next method;
        #  3) if no more methods are available, quit.

        if self.currentMethod in self.fallbackMethods:
            # remove the already-used method to avoid infinite looping between methods
            self.fallbackMethods.remove(self.currentMethod)

        assert (self.currentMethod in self.methods)
        self.methods.remove(self.currentMethod)

        anyNotFound = any([math.isinf(r.ofValue) for r in self.runners])
        if anyNotFound or self.isUsingFallback:
            if len(self.fallbackMethods) == 0:
                if anyNotFound:
                    g.log(LOG_INFO, "terminating {}: failed to evaluate the objective function".format(self.getName()))
                else:
                    g.log(LOG_INFO, "terminating {}: all fallback methods exhausted without reaching consensus".format(self.getName()))
                self.pool.finishJob(self)
                return

            self.pastMethods.append(self.currentMethod)
            if bool(g.getConfig("optimization.randomizeMethodSelection")):
                self.currentMethod = random.choice(self.fallbackMethods)
            else:
                self.currentMethod = self.fallbackMethods[0]
            # make sure the fallback methods are also in methods
            if self.currentMethod not in self.methods:
                self.methods.append(self.currentMethod)
            g.log(LOG_INFO, "switching {} to a fallback method {}".format(self.getName(), self.currentMethod))
            self.isUsingFallback = True
            # switch to the fallback method
            if not self.createRunners():
                self.pool.finishJob(self)
            return

        if len(self.methods) == 0:
            g.log(LOG_INFO, "terminating {}: all methods exhausted without reaching consensus".format(self.getName()))
            self.pool.finishJob(self)
            return

        # go for the next method
        self.pastMethods.append(self.currentMethod)
        if bool(g.getConfig("optimization.randomizeMethodSelection")):
            self.currentMethod = random.choice(self.methods)
        else:
            self.currentMethod = self.methods[0]

        g.log(LOG_INFO, "switching {} to the next method {}".format(
            self.getName(), self.currentMethod))
        if not self.createRunners():
            self.pool.finishJob(self)
            return