def test_cfunc(self):
with self.assertRaises(TypeError):
func = CThreadPool.lookupCFunction("WRONG-TYPE",
"no-this-does-not-exist")
with self.assertRaises(AttributeError):
func = CThreadPool.lookupCFunction(TEST_LIB,
"no-this-does-not-exist")
def __init__(self, ert, size, verbose=False):
self._ert = ert
""" :type: res.enkf.EnKFMain """
self._size = size
max_runtime = ert.analysisConfig().get_max_runtime()
job_queue = ert.get_queue_config().create_job_queue()
self._queue_manager = JobQueueManager(job_queue)
self._queue_manager.startQueue(size, verbose=verbose)
self._run_args = {}
""" :type: dict[int, RunArg] """
self._thread_pool = CThreadPool(8)
self._thread_pool.addTaskFunction("submitJob", ENKF_LIB,
"enkf_main_isubmit_job__")
def test_context(self):
N = 256
arg = ctypes.c_int(0)
job = CThreadPool.lookupCFunction(TEST_LIB, "thread_pool_test_func1")
with startCThreadPool(16) as tp:
for i in range(N):
tp.addTask(job, ctypes.byref(arg))
self.assertEqual(arg.value, N)
def __init__(self, ert, size, verbose=False):
self._ert = ert
""" :type: res.enkf.EnKFMain """
self._size = size
max_runtime = ert.analysisConfig().get_max_runtime()
raise Exception(
"Code has lost access to job_queue instance. Refactor required.")
job_queue = None
self._queue_manager = JobQueueManager(job_queue)
self._queue_manager.startQueue(size, verbose=verbose)
self._run_args = {}
""" :type: dict[int, RunArg] """
self._thread_pool = CThreadPool(8)
self._thread_pool.addTaskFunction("submitJob", ENKF_LIB,
"enkf_main_isubmit_job__")
def __init__(self, ert, init_fs, result_fs, mask, itr , verbose=False):
self._ert = ert
""" :type: res.enkf.EnKFMain """
max_runtime = ert.analysisConfig().get_max_runtime()
self._mask = mask
job_queue = ert.get_queue_config().create_job_queue()
self._queue_manager = JobQueueManager(job_queue)
self._queue_manager.startQueue( mask.count( ), verbose=verbose)
self._run_args = {}
""" :type: dict[int, RunArg] """
self._thread_pool = CThreadPool(8)
self._thread_pool.addTaskFunction("submitJob", ENKF_LIB, "enkf_main_isubmit_job__")
subst_list = self._ert.getDataKW( )
path_fmt = self._ert.getModelConfig().getRunpathFormat()
self._run_context = ErtRunContext( EnkfRunType.ENSEMBLE_EXPERIMENT, init_fs, result_fs, None, mask, path_fmt, subst_list, itr)
def test_add_task_function(self):
pool = CThreadPool(32, start=True)
pool.addTaskFunction("testFunction", TEST_LIB, "thread_pool_test_func1")
arg = ctypes.c_int(0)
task_count = 256
for i in range(task_count):
pool.testFunction(ctypes.byref(arg))
pool.join()
self.assertEqual(arg.value, task_count)
def test_create(self):
pool = CThreadPool(32, start=True)
job = CThreadPool.lookupCFunction(TEST_LIB, "thread_pool_test_func1")
arg = ctypes.c_int(0)
N = 256
for i in range(N):
pool.addTask(job, ctypes.byref(arg))
pool.join()
self.assertEqual(arg.value, N)
class SimulationContext(object):
def __init__(self, ert, size, verbose=False):
self._ert = ert
""" :type: res.enkf.EnKFMain """
self._size = size
max_runtime = ert.analysisConfig().get_max_runtime()
raise Exception(
"Code has lost access to job_queue instance. Refactor required.")
job_queue = None
self._queue_manager = JobQueueManager(job_queue)
self._queue_manager.startQueue(size, verbose=verbose)
self._run_args = {}
""" :type: dict[int, RunArg] """
self._thread_pool = CThreadPool(8)
self._thread_pool.addTaskFunction("submitJob", ENKF_LIB,
"enkf_main_isubmit_job__")
def addSimulation(self, iens, target_fs):
if iens >= self._size:
raise UserWarning("Realization number out of range: %d >= %d" %
(iens, self._size))
if iens in self._run_args:
raise UserWarning("Realization number: '%d' already queued" % iens)
runpath_fmt = self._ert.getModelConfig().getRunpathFormat()
member = self._ert.getRealisation(iens)
runpath = ErtRunContext.createRunpath(iens, runpath_fmt,
member.getDataKW())
run_arg = RunArg.createEnsembleExperimentRunArg(
target_fs, iens, runpath)
self._ert.createRunPath(run_arg)
self._run_args[iens] = run_arg
self._thread_pool.submitJob(ArgPack(self._ert, run_arg))
def isRunning(self):
return self._queue_manager.isRunning()
def getNumRunning(self):
return self._queue_manager.getNumRunning()
def getNumSuccess(self):
return self._queue_manager.getNumSuccess()
def getNumFailed(self):
return self._queue_manager.getNumFailed()
def getNumWaiting(self):
return self._queue_manager.getNumWaiting()
def didRealizationSucceed(self, iens):
queue_index = self._run_args[iens].getQueueIndex()
return self._queue_manager.didJobSucceed(queue_index)
def didRealizationFail(self, iens):
# For the purposes of this class, a failure should be anything (killed job, etc) that is not an explicit success.
return not self.didRealizationSucceed(iens)
def isRealizationQueued(self, iens):
return iens in self._run_args
def isRealizationFinished(self, iens):
run_arg = self._run_args[iens]
if run_arg.isSubmitted():
queue_index = run_arg.getQueueIndex()
return self._queue_manager.isJobComplete(queue_index)
else:
return False
def __repr__(self):
running = 'running' if self.isRunning() else 'not running'
numRunn = self.getNumRunning()
numSucc = self.getNumSuccess()
numFail = self.getNumFailed()
numWait = self.getNumWaiting()
fmt = '%s, #running = %d, #success = %d, #failed = %d, #waiting = %d'
fmt = fmt % (running, numRunn, numSucc, numFail, numWait)
return 'SimulationContext(%s)' % fmt
class SimulationContext(object):
def __init__(self, ert, sim_fs, mask, itr, verbose=False):
self._ert = ert
""" :type: res.enkf.EnKFMain """
max_runtime = ert.analysisConfig().get_max_runtime()
self._mask = mask
job_queue = ert.get_queue_config().create_job_queue()
self._queue_manager = JobQueueManager(job_queue)
self._queue_manager.startQueue(mask.count(), verbose=verbose)
self._run_args = {}
""" :type: dict[int, RunArg] """
self._thread_pool = CThreadPool(8)
self._thread_pool.addTaskFunction("submitJob", ENKF_LIB,
"enkf_main_isubmit_job__")
subst_list = self._ert.getDataKW()
path_fmt = self._ert.getModelConfig().getRunpathFormat()
jobname_fmt = self._ert.getModelConfig().getJobnameFormat()
self._run_context = ErtRunContext(EnkfRunType.ENSEMBLE_EXPERIMENT,
sim_fs, None, mask, path_fmt,
jobname_fmt, subst_list, itr)
self._ert.createRunpath(self._run_context)
def __len__(self):
return self._mask.count()
def addSimulation(self, iens, geo_id):
if not (0 <= iens < len(self._run_context)):
raise UserWarning("Realization number out of range: %d >= %d" %
(iens, len(self._run_context)))
if not self._mask[iens]:
raise UserWarning("Realization number: '%d' is not active" % iens)
if iens in self._run_args:
raise UserWarning("Realization number: '%d' already queued" % iens)
run_arg = self._run_context[iens]
queue = self._queue_manager.get_job_queue()
self._run_args[iens] = run_arg
self._thread_pool.submitJob(ArgPack(self._ert, run_arg, queue, geo_id))
def isRunning(self):
return self._queue_manager.isRunning()
def getNumPending(self):
return self._queue_manager.getNumPending()
def getNumRunning(self):
return self._queue_manager.getNumRunning()
def getNumSuccess(self):
return self._queue_manager.getNumSuccess()
def getNumFailed(self):
return self._queue_manager.getNumFailed()
def getNumWaiting(self):
return self._queue_manager.getNumWaiting()
def didRealizationSucceed(self, iens):
queue_index = self._run_args[iens].getQueueIndex()
return self._queue_manager.didJobSucceed(queue_index)
def didRealizationFail(self, iens):
# For the purposes of this class, a failure should be anything (killed job, etc) that is not an explicit success.
return not self.didRealizationSucceed(iens)
def isRealizationQueued(self, iens):
return iens in self._run_args
def isRealizationFinished(self, iens):
run_arg = self._run_args[iens]
if run_arg.isSubmitted():
queue_index = run_arg.getQueueIndex()
return self._queue_manager.isJobComplete(queue_index)
else:
return False
def __repr__(self):
running = 'running' if self.isRunning() else 'not running'
numRunn = self.getNumRunning()
numSucc = self.getNumSuccess()
numFail = self.getNumFailed()
numWait = self.getNumWaiting()
fmt = '%s, #running = %d, #success = %d, #failed = %d, #waiting = %d'
fmt = fmt % (running, numRunn, numSucc, numFail, numWait)
return 'SimulationContext(%s)' % fmt
def get_sim_fs(self):
return self._run_context.get_sim_fs()
def get_run_context(self):
return self._run_context