def _simulation_run_status(data, quiet=False):
"""Look for simulation status and output
Args:
data (dict): request
quiet (bool): don't write errors to log
Returns:
dict: status response
"""
try:
#TODO(robnagler): Lock
rep = simulation_db.report_info(data)
is_processing = runner.job_is_processing(rep.job_id)
is_running = rep.job_status in _RUN_STATES
res = {'state': rep.job_status}
pkdc(
'{}: is_processing={} is_running={} state={} cached_data={}',
rep.job_id,
is_processing,
is_running,
rep.job_status,
bool(rep.cached_data),
)
if is_processing and not is_running:
runner.job_race_condition_reap(rep.job_id)
pkdc('{}: is_processing and not is_running', rep.job_id)
is_processing = False
template = sirepo.template.import_module(data)
if is_processing:
if not rep.cached_data:
return _simulation_error(
'input file not found, but job is running',
rep.input_file,
)
else:
is_running = False
if rep.run_dir.exists():
if hasattr(template,
'prepare_output_file') and 'models' in data:
template.prepare_output_file(rep.run_dir, data)
res2, err = simulation_db.read_result(rep.run_dir)
if err:
if simulation_db.is_parallel(data):
# allow parallel jobs to use template to parse errors below
res['state'] = 'error'
else:
if hasattr(template, 'parse_error_log'):
res = template.parse_error_log(rep.run_dir)
if res:
return res
return _simulation_error(err, 'error in read_result',
rep.run_dir)
else:
res = res2
if simulation_db.is_parallel(data):
new = template.background_percent_complete(
rep.model_name,
rep.run_dir,
is_running,
)
new.setdefault('percentComplete', 0.0)
new.setdefault('frameCount', 0)
res.update(new)
res['parametersChanged'] = rep.parameters_changed
if res['parametersChanged']:
pkdlog(
'{}: parametersChanged=True req_hash={} cached_hash={}',
rep.job_id,
rep.req_hash,
rep.cached_hash,
)
#TODO(robnagler) verify serial number to see what's newer
res.setdefault('startTime', _mtime_or_now(rep.input_file))
res.setdefault('lastUpdateTime', _mtime_or_now(rep.run_dir))
res.setdefault('elapsedTime', res['lastUpdateTime'] - res['startTime'])
if is_processing:
res['nextRequestSeconds'] = simulation_db.poll_seconds(
rep.cached_data)
res['nextRequest'] = {
'report': rep.model_name,
'reportParametersHash': rep.cached_hash,
'simulationId': rep.cached_data['simulationId'],
'simulationType': rep.cached_data['simulationType'],
}
pkdc(
'{}: processing={} state={} cache_hit={} cached_hash={} data_hash={}',
rep.job_id,
is_processing,
res['state'],
rep.cache_hit,
rep.cached_hash,
rep.req_hash,
)
except Exception:
return _simulation_error(pkdexc(), quiet=quiet)
return res
def _simulation_run_status(data, quiet=False):
"""Look for simulation status and output
Args:
data (dict): request
quiet (bool): don't write errors to log
Returns:
dict: status response
"""
try:
#TODO(robnagler): Lock
rep = simulation_db.report_info(data)
is_processing = cfg.job_queue.is_processing(rep.job_id)
is_running = rep.job_status in _RUN_STATES
res = {'state': rep.job_status}
pkdc(
'{}: is_processing={} is_running={} state={} cached_data={}',
rep.job_id,
is_processing,
is_running,
rep.job_status,
bool(rep.cached_data),
)
if is_processing and not is_running:
cfg.job_queue.race_condition_reap(rep.job_id)
pkdc('{}: is_processing and not is_running', rep.job_id)
is_processing = False
if is_processing:
if not rep.cached_data:
return _simulation_error(
'input file not found, but job is running',
rep.input_file,
)
else:
is_running = False
if rep.run_dir.exists():
res, err = simulation_db.read_result(rep.run_dir)
if err:
return _simulation_error(err, 'error in read_result', rep.run_dir)
if simulation_db.is_parallel(data):
template = sirepo.template.import_module(data)
new = template.background_percent_complete(
rep.model_name,
rep.run_dir,
is_running,
simulation_db.get_schema(data['simulationType']),
)
new.setdefault('percentComplete', 0.0)
new.setdefault('frameCount', 0)
res.update(new)
res['parametersChanged'] = rep.parameters_changed
if res['parametersChanged']:
pkdlog(
'{}: parametersChanged=True req_hash={} cached_hash={}',
rep.job_id,
rep.req_hash,
rep.cached_hash,
)
#TODO(robnagler) verify serial number to see what's newer
res.setdefault('startTime', _mtime_or_now(rep.input_file))
res.setdefault('lastUpdateTime', _mtime_or_now(rep.run_dir))
res.setdefault('elapsedTime', res['lastUpdateTime'] - res['startTime'])
if is_processing:
res['nextRequestSeconds'] = simulation_db.poll_seconds(rep.cached_data)
res['nextRequest'] = {
'report': rep.model_name,
'reportParametersHash': rep.cached_hash,
'simulationId': rep.cached_data['simulationId'],
'simulationType': rep.cached_data['simulationType'],
}
pkdc(
'{}: processing={} state={} cache_hit={} cached_hash={} data_hash={}',
rep.job_id,
is_processing,
res['state'],
rep.cache_hit,
rep.cached_hash,
rep.req_hash,
)
except Exception:
return _simulation_error(pkdexc(), quiet=quiet)
return res
def _simulation_run_status_runner_daemon(data, quiet=False):
"""Look for simulation status and output
Args:
data (dict): request
quiet (bool): don't write errors to log
Returns:
dict: status response
"""
try:
run_dir = simulation_db.simulation_run_dir(data)
jhash = template_common.report_parameters_hash(data)
status = runner_client.report_job_status(run_dir, jhash)
is_running = status is runner_client.JobStatus.RUNNING
rep = simulation_db.report_info(data)
res = {'state': status.value}
if not is_running:
if status is not runner_client.JobStatus.MISSING:
res, err = runner_client.run_extract_job(
run_dir,
jhash,
'result',
data,
)
if err:
return _simulation_error(err, 'error in read_result',
run_dir)
if simulation_db.is_parallel(data):
new = runner_client.run_extract_job(
run_dir,
jhash,
'background_percent_complete',
is_running,
)
new.setdefault('percentComplete', 0.0)
new.setdefault('frameCount', 0)
res.update(new)
res['parametersChanged'] = rep.parameters_changed
if res['parametersChanged']:
pkdlog(
'{}: parametersChanged=True req_hash={} cached_hash={}',
rep.job_id,
rep.req_hash,
rep.cached_hash,
)
#TODO(robnagler) verify serial number to see what's newer
res.setdefault('startTime', _mtime_or_now(rep.input_file))
res.setdefault('lastUpdateTime', _mtime_or_now(rep.run_dir))
res.setdefault('elapsedTime', res['lastUpdateTime'] - res['startTime'])
if is_running:
res['nextRequestSeconds'] = simulation_db.poll_seconds(
rep.cached_data)
res['nextRequest'] = {
'report': rep.model_name,
'reportParametersHash': rep.cached_hash,
'simulationId': rep.cached_data['simulationId'],
'simulationType': rep.cached_data['simulationType'],
}
pkdc(
'{}: processing={} state={} cache_hit={} cached_hash={} data_hash={}',
rep.job_id,
is_running,
res['state'],
rep.cache_hit,
rep.cached_hash,
rep.req_hash,
)
except Exception:
return _simulation_error(pkdexc(), quiet=quiet)
return res