示例#1
0
    def test_cfunc(self):
        with self.assertRaises(TypeError):
            func = CThreadPool.lookupCFunction("WRONG-TYPE",
                                               "no-this-does-not-exist")

        with self.assertRaises(AttributeError):
            func = CThreadPool.lookupCFunction(TEST_LIB,
                                               "no-this-does-not-exist")
示例#2
0
    def __init__(self, ert, size, verbose=False):
        self._ert = ert
        """ :type: ert.enkf.EnKFMain """
        self._size = size

        self._queue_manager = JobQueueManager(ert.siteConfig().getJobQueue())
        self._queue_manager.startQueue(size, verbose=verbose)
        self._run_args = {}
        """ :type: dict[int, RunArg] """

        self._thread_pool = CThreadPool(8)
        self._thread_pool.addTaskFunction("submitJob", ENKF_LIB,
                                          "enkf_main_isubmit_job__")
示例#3
0
    def __init__(self , ert_handle , size , run_fs , run_count):
        self.ert_handle = ert_handle
        self.size = size
        self.runner = ert_handle.getEnkfSimulationRunner()
        
        site_config = self.ert_handle.siteConfig()
        self.queue_manager = JobQueueManager( site_config.getJobQueue() )
        self.queue_manager.startQueue( size , verbose = True )

        mask = BoolVector( default_value = True )
        mask[size - 1] = True
        
        self.ert_handle.addDataKW("<ELCO_RUN_COUNT>" , "%s" % run_count)
        self.ert_run_context = self.ert_handle.getRunContextENSEMPLE_EXPERIMENT( run_fs , mask )
        self.thread_pool = CThreadPool( 8 )
        self.submit_func = CThreadPool.lookupCFunction( ENKF_LIB , "enkf_main_isubmit_job__" )
示例#4
0
 def test_context(self):
     N = 256
     arg = ctypes.c_int(0)
     job = CThreadPool.lookupCFunction(TEST_LIB, "thread_pool_test_func1")
     with startCThreadPool(16) as tp:
         for i in range(N):
             tp.addTask(job, ctypes.byref(arg))
     self.assertEqual(arg.value, N)
示例#5
0
 def test_context(self):
     N = 256
     arg = ctypes.c_int(0)
     job = CThreadPool.lookupCFunction(TEST_LIB, "thread_pool_test_func1")
     with startCThreadPool(16) as tp:
         for i in range(N):
             tp.addTask(job, ctypes.byref(arg))
     self.assertEqual(arg.value, N)
示例#6
0
    def test_add_task_function(self):
        pool = CThreadPool(32, start=True)
        pool.addTaskFunction("testFunction", TEST_LIB,
                             "thread_pool_test_func1")

        arg = ctypes.c_int(0)
        task_count = 256
        for i in range(task_count):
            pool.testFunction(ctypes.byref(arg))

        pool.join()
        self.assertEqual(arg.value, task_count)
示例#7
0
    def __init__(self, ert, size, verbose=False):
        self._ert = ert
        """ :type: ert.enkf.EnKFMain """
        self._size = size

        self._queue_manager = JobQueueManager(ert.siteConfig().getJobQueue())
        self._queue_manager.startQueue(size, verbose=verbose)

        mask = BoolVector(default_value=True, initial_size=size)
        runpath_fmt = self._ert.getModelConfig().getRunpathFormat()
        subst_list = self._ert.getDataKW()
        self._runpath_list = ErtRunContext.createRunpathList(
            mask, runpath_fmt, subst_list)

        self._run_args = {}
        """ :type: dict[int, RunArg] """

        self._thread_pool = CThreadPool(8)
        self._thread_pool.addTaskFunction("submitJob", ENKF_LIB,
                                          "enkf_main_isubmit_job__")
示例#8
0
    def test_create(self):
        pool = CThreadPool(32, start=True)
        job = CThreadPool.lookupCFunction(TEST_LIB, "thread_pool_test_func1")
        arg = ctypes.c_int(0)

        N = 256
        for i in range(N):
            pool.addTask(job, ctypes.byref(arg))
        pool.join()
        self.assertEqual(arg.value, N)
示例#9
0
    def __init__(self, ert, size, verbose=False):
        self._ert = ert
        """ :type: ert.enkf.EnKFMain """
        self._size = size

        self._queue_manager = JobQueueManager(ert.siteConfig().getJobQueue())
        self._queue_manager.startQueue(size, verbose=verbose)
        self._run_args = {}
        """ :type: dict[int, RunArg] """

        self._thread_pool = CThreadPool(8)
        self._thread_pool.addTaskFunction("submitJob", ENKF_LIB, "enkf_main_isubmit_job__")
示例#10
0
    def test_add_task_function(self):
        pool = CThreadPool(32, start=True)
        pool.addTaskFunction("testFunction", TEST_LIB, "thread_pool_test_func1")

        arg = ctypes.c_int(0)
        task_count = 256
        for i in range(task_count):
            pool.testFunction(ctypes.byref(arg))

        pool.join()
        self.assertEqual(arg.value, task_count)
示例#11
0
    def test_create(self):
        pool = CThreadPool(32, start=True)
        job = CThreadPool.lookupCFunction(TEST_LIB, "thread_pool_test_func1")
        arg = ctypes.c_int(0)

        N = 256
        for i in range(N):
            pool.addTask(job, ctypes.byref(arg))
        pool.join()
        self.assertEqual(arg.value, N)
示例#12
0
    def __init__(self, ert, size, verbose=False):
        self._ert = ert
        """ :type: ert.enkf.EnKFMain """
        self._size = size

        self._queue_manager = JobQueueManager(ert.siteConfig().getJobQueue())
        self._queue_manager.startQueue(size, verbose=verbose)

        mask = BoolVector(default_value=True, initial_size=size)
        runpath_fmt = self._ert.getModelConfig().getRunpathFormat()
        subst_list = self._ert.getDataKW()
        self._runpath_list = ErtRunContext.createRunpathList(mask, runpath_fmt, subst_list)

        self._run_args = {}
        """ :type: dict[int, RunArg] """

        self._thread_pool = CThreadPool(8)
        self._thread_pool.addTaskFunction("submitJob", ENKF_LIB, "enkf_main_isubmit_job__")
示例#13
0
class SimulationContext(object):
    def __init__(self, ert, size, verbose=False):
        self._ert = ert
        """ :type: ert.enkf.EnKFMain """
        self._size = size
        
        max_runtime = ert.analysisConfig().get_max_runtime()
        job_queue = ert.siteConfig().getJobQueue()
        job_queue.set_max_job_duration(max_runtime)

        self._queue_manager = JobQueueManager(job_queue)
        self._queue_manager.startQueue(size, verbose=verbose)
        self._run_args = {}
        """ :type: dict[int, RunArg] """

        self._thread_pool = CThreadPool(8)
        self._thread_pool.addTaskFunction("submitJob", ENKF_LIB, "enkf_main_isubmit_job__")


    def addSimulation(self, iens, target_fs):
        if iens >= self._size:
            raise UserWarning("Realization number out of range: %d >= %d" % (iens, self._size))

        if iens in self._run_args:
            raise UserWarning("Realization number: '%d' already queued" % iens)

        runpath_fmt = self._ert.getModelConfig().getRunpathFormat()
        member = self._ert.getRealisation(iens)
        runpath = ErtRunContext.createRunpath(iens , runpath_fmt, member.getDataKW( ))
        run_arg = RunArg.createEnsembleExperimentRunArg(target_fs, iens, runpath)

        self._ert.createRunPath(run_arg)

        self._run_args[iens] = run_arg
        self._thread_pool.submitJob(ArgPack(self._ert, run_arg))


    def isRunning(self):
        return self._queue_manager.isRunning()


    def getNumRunning(self):
        return self._queue_manager.getNumRunning()


    def getNumSuccess(self):
        return self._queue_manager.getNumSuccess()


    def getNumFailed(self):
        return self._queue_manager.getNumFailed()

    def getNumWaiting(self):
        return self._queue_manager.getNumWaiting()


    def didRealizationSucceed(self, iens):
        queue_index = self._run_args[iens].getQueueIndex()
        return self._queue_manager.didJobSucceed(queue_index)


    def didRealizationFail(self, iens):
        # For the purposes of this class, a failure should be anything (killed job, etc) that is not an explicit success.
        return not self.didRealizationSucceed(iens)


    def isRealizationQueued(self, iens):
        return iens in self._run_args


    def isRealizationFinished(self, iens):
        run_arg = self._run_args[iens]

        if run_arg.isSubmitted():
            queue_index = run_arg.getQueueIndex()
            return self._queue_manager.isJobComplete(queue_index)
        else:
            return False

    def __repr__(self):
        running = 'running' if self.isRunning() else 'not running'
        numRunn = self.getNumRunning()
        numSucc = self.getNumSuccess()
        numFail = self.getNumFailed()
        numWait = self.getNumWaiting()
        fmt = '%s, #running = %d, #success = %d, #failed = %d, #waiting = %d'
        fmt =  fmt % (running, numRunn, numSucc, numFail, numWait)
        return 'SimulationContext(%s)' % fmt
示例#14
0
class SimulationContext(object):
    def __init__(self, ert, size, verbose=False):
        self._ert = ert
        """ :type: ert.enkf.EnKFMain """
        self._size = size

        self._queue_manager = JobQueueManager(ert.siteConfig().getJobQueue())
        self._queue_manager.startQueue(size, verbose=verbose)
        self._run_args = {}
        """ :type: dict[int, RunArg] """

        self._thread_pool = CThreadPool(8)
        self._thread_pool.addTaskFunction("submitJob", ENKF_LIB,
                                          "enkf_main_isubmit_job__")

    def addSimulation(self, iens, target_fs):
        if iens >= self._size:
            raise UserWarning("Realization number out of range: %d >= %d" %
                              (iens, self._size))

        if iens in self._run_args:
            raise UserWarning("Realization number: '%d' already queued" % iens)

        runpath_fmt = self._ert.getModelConfig().getRunpathFormat()
        member = self._ert.getRealisation(iens)
        runpath = ErtRunContext.createRunpath(iens, runpath_fmt,
                                              member.getDataKW())
        run_arg = RunArg.createEnsembleExperimentRunArg(
            target_fs, iens, runpath)

        self._ert.createRunPath(run_arg)

        self._run_args[iens] = run_arg
        self._thread_pool.submitJob(ArgPack(self._ert, run_arg))

    def isRunning(self):
        return self._queue_manager.isRunning()

    def getNumRunning(self):
        return self._queue_manager.getNumRunning()

    def getNumSuccess(self):
        return self._queue_manager.getNumSuccess()

    def getNumFailed(self):
        return self._queue_manager.getNumFailed()

    def getNumWaiting(self):
        return self._queue_manager.getNumWaiting()

    def didRealizationSucceed(self, iens):
        queue_index = self._run_args[iens].getQueueIndex()
        return self._queue_manager.didJobSucceed(queue_index)

    def didRealizationFail(self, iens):
        queue_index = self._run_args[iens].getQueueIndex()
        return self._queue_manager.didJobFail(queue_index)

    def isRealizationQueued(self, iens):
        return iens in self._run_args

    def isRealizationFinished(self, iens):
        run_arg = self._run_args[iens]

        if run_arg.isSubmitted():
            queue_index = run_arg.getQueueIndex()
            return self._queue_manager.isJobComplete(queue_index)
        else:
            return False
示例#15
0
class SimulationContext(object):
    def __init__(self, ert, size, verbose=False):
        self._ert = ert
        """ :type: ert.enkf.EnKFMain """
        self._size = size

        self._queue_manager = JobQueueManager(ert.siteConfig().getJobQueue())
        self._queue_manager.startQueue(size, verbose=verbose)
        self._run_args = {}
        """ :type: dict[int, RunArg] """

        self._thread_pool = CThreadPool(8)
        self._thread_pool.addTaskFunction("submitJob", ENKF_LIB, "enkf_main_isubmit_job__")

    def addSimulation(self, iens, target_fs):
        if iens >= self._size:
            raise UserWarning("Realization number out of range: %d >= %d" % (iens, self._size))

        if iens in self._run_args:
            raise UserWarning("Realization number: '%d' already queued" % iens)

        runpath_fmt = self._ert.getModelConfig().getRunpathFormat()
        member = self._ert.getRealisation(iens)
        runpath = ErtRunContext.createRunpath(iens, runpath_fmt, member.getDataKW())
        run_arg = RunArg.createEnsembleExperimentRunArg(target_fs, iens, runpath)
        self._run_args[iens] = run_arg
        self._thread_pool.submitJob(ArgPack(self._ert, run_arg))

    def isRunning(self):
        return self._queue_manager.isRunning()

    def getNumRunning(self):
        return self._queue_manager.getNumRunning()

    def getNumSuccess(self):
        return self._queue_manager.getNumSuccess()

    def getNumFailed(self):
        return self._queue_manager.getNumFailed()

    def getNumWaiting(self):
        return self._queue_manager.getNumWaiting()

    def didRealizationSucceed(self, iens):
        queue_index = self._run_args[iens].getQueueIndex()
        return self._queue_manager.didJobSucceed(queue_index)

    def didRealizationFail(self, iens):
        queue_index = self._run_args[iens].getQueueIndex()
        return self._queue_manager.didJobFail(queue_index)

    def isRealizationQueued(self, iens):
        return iens in self._run_args

    def isRealizationFinished(self, iens):
        run_arg = self._run_args[iens]

        if run_arg.isSubmitted():
            queue_index = run_arg.getQueueIndex()
            return self._queue_manager.isJobComplete(queue_index)
        else:
            return False
示例#16
0
    def test_cfunc(self):
        with self.assertRaises(TypeError):
            func = CThreadPool.lookupCFunction("WRONG-TYPE", "no-this-does-not-exist")

        with self.assertRaises(AttributeError):
            func = CThreadPool.lookupCFunction(TEST_LIB, "no-this-does-not-exist")
示例#17
0
class RunContext(object):
    def __init__(self , ert_handle , size , run_fs , run_count):
        self.ert_handle = ert_handle
        self.size = size
        self.runner = ert_handle.getEnkfSimulationRunner()
        
        site_config = self.ert_handle.siteConfig()
        self.queue_manager = JobQueueManager( site_config.getJobQueue() )
        self.queue_manager.startQueue( size , verbose = True )

        mask = BoolVector( default_value = True )
        mask[size - 1] = True
        
        self.ert_handle.addDataKW("<ELCO_RUN_COUNT>" , "%s" % run_count)
        self.ert_run_context = self.ert_handle.getRunContextENSEMPLE_EXPERIMENT( run_fs , mask )
        self.thread_pool = CThreadPool( 8 )
        self.submit_func = CThreadPool.lookupCFunction( ENKF_LIB , "enkf_main_isubmit_job__" )

        
    def isRunning(self):
        return self.queue_manager.isRunning()


    def getNumRunning(self):
        return self.queue_manager.getNumRunning()


    def getNumSuccess(self):
        return self.queue_manager.getNumSuccess()

    
    def getNumFailed(self):
        return self.queue_manager.getNumFailed()

    
    def startSimulation(self , iens):
        member_context = self.ert_run_context.iensGet( iens )
        arg = ArgPack( self.ert_handle , member_context )
        self.thread_pool.addTask( self.submit_func , arg )
        

    def realisationSuccess(self, iens):
        run_arg = self.ert_run_context.iensGet( iens )
        queue_index = run_arg.getQueueIndex()
        return self.queue_manager.jobSuccess( queue_index )

    
    def realisationFailed(self, iens):
        run_arg = self.ert_run_context.iensGet( iens )
        queue_index = run_arg.getQueueIndex()
        return self.queue_manager.jobFailed( queue_index )


    # Running is slightly misleading; we have three catogories:
    # Complete, Failed, and "Running". Running in this context is
    # actually the 'the rest', and includes jobs in all sorts of
    # waiting.
    
    def realisationRunning(self, iens):
        run_arg = self.ert_run_context.iensGet( iens )
        if run_arg.isSubmitted():
            queue_index = run_arg.getQueueIndex()
            if self.queue_manager.jobRunning( queue_index ) or self.queue_manager.jobWaiting( queue_index ):
                return True
            else:
                return False
        else:
            return True
        

    def exportRunpathList(self):
        runpath_list = self.ert_handle.getRunpathList( )
        runpath_list.export( )