def updateMask(self, gen_data_config, report_step, fs, active_mask,
subst_list):
run_arg = RunArg.createEnsembleExperimentRunArg(
"run_id", fs, 0, "Path", "jobname", subst_list)
load_context = ForwardLoadContext(run_arg=run_arg,
report_step=report_step)
self._update_active_mask(gen_data_config, load_context, active_mask)
def update_progress_for_index(self, iteration: int, idx,
run_arg: RunArg) -> None:
try:
# will throw if not yet submitted (is in a limbo state)
queue_index = run_arg.getQueueIndex()
except (ValueError, AttributeError):
return
status = None
timed_out = False
if self._job_queue:
status = self._job_queue.getJobStatus(queue_index)
timed_out = self._job_queue.did_job_time_out(queue_index)
# Avoids reading from disk for jobs in these states since there's no
# data anyway. If timed out, never exit here as that would prevent
# propagation of the failure status.
if (status in [
JobStatusType.JOB_QUEUE_PENDING,
JobStatusType.JOB_QUEUE_SUBMITTED,
JobStatusType.JOB_QUEUE_WAITING,
] and not timed_out):
return
fms = self.realization_progress[iteration].get(run_arg.iens, None)
jobs = fms[0] if fms else None
# Don't load from file if you are finished
if not fms or not BaseRunModel.is_forward_model_finished(fms[0]):
loaded = ForwardModelStatus.load(run_arg.runpath, num_retry=1)
if not loaded and not timed_out:
# If this idx timed out, returning here would prevent
# non-successful jobs in being marked as failed (timed out). So
# return only in the case where it did not time out.
return
if loaded:
jobs = loaded.jobs
if timed_out:
for job in jobs:
if job.status != "Success":
job.error = "The run is cancelled due to reaching MAX_RUNTIME"
job.status = "Failure"
self.realization_progress[iteration][run_arg.iens] = jobs, status
# Fetch out the job_queue from the SiteConfig object. In addition we
# create a JobQueueManager objects which wraps the queue. The purpose
# of this manager object is to let the queue run nonblocking in the
# background.
site_config = ert.siteConfig( )
queue_manager = JobQueueManager( site_config.getJobQueue( ) )
queue_manager.startQueue( ert.getEnsembleSize( ) , verbose = False )
# Create list of RunArg instances which hold metadata for one running
# realisation, create the directory where the simulation should run
# and submit the simulation.
path_fmt = "/tmp/run%d"
arg_list = [ RunArg.createEnsembleExperimentRunArg(fs, iens, path_fmt % iens) for iens in range(ert.getEnsembleSize()) ]
for arg in arg_list:
ert.createRunPath( arg )
ert.submitSimulation( arg )
while True:
print("Waiting:%d Running:%d Complete:%d/%d" % (queue_manager.getNumWaiting( ), queue_manager.getNumRunning( ) , queue_manager.getNumSuccess() , queue_manager.getNumFailed( )))
if not queue_manager.isRunning( ):
break
time.sleep( 5 )
ens_config = ert.ensembleConfig( )
data_config = ens_config["SNAKE_OIL_OPR_DIFF"]
param_config = ens_config["SNAKE_OIL_PARAM"]
realisation.initialize(fs)
# Fetch out the job_queue from the SiteConfig object. In addition we
# create a JobQueueManager objects which wraps the queue. The purpose
# of this manager object is to let the queue run nonblocking in the
# background.
site_config = ert.siteConfig()
queue_manager = JobQueueManager(site_config.getJobQueue())
queue_manager.startQueue(ert.getEnsembleSize(), verbose=False)
# Create list of RunArg instances which hold metadata for one running
# realisation, create the directory where the simulation should run
# and submit the simulation.
path_fmt = "/tmp/run%d"
arg_list = [
RunArg.createEnsembleExperimentRunArg(fs, iens, path_fmt % iens)
for iens in range(ert.getEnsembleSize())
]
for arg in arg_list:
ert.createRunPath(arg)
ert.submitSimulation(arg)
while True:
print("Waiting:%d Running:%d Complete:%d/%d" %
(queue_manager.getNumWaiting(), queue_manager.getNumRunning(),
queue_manager.getNumSuccess(), queue_manager.getNumFailed()))
if not queue_manager.isRunning():
break
time.sleep(5)