def insert_db_job(self, row_idx, row):
"""Load job element from DB post restart."""
if row_idx == 0:
LOG.info("LOADING job data")
(point_string, name, status, submit_num, time_submit, time_run,
time_run_exit, batch_sys_name, batch_sys_job_id, platform_name) = row
if status not in JOB_STATUS_SET:
return
t_id = f'{self.workflow_id}{ID_DELIM}{point_string}{ID_DELIM}{name}'
j_id = f'{t_id}{ID_DELIM}{submit_num}'
try:
tdef = self.schd.config.get_taskdef(name)
j_owner = self.schd.owner
if platform_name:
j_host = get_host_from_platform(get_platform(platform_name))
else:
j_host = self.schd.host
j_buf = PbJob(
stamp=f'{j_id}@{time()}',
id=j_id,
submit_num=submit_num,
state=status,
task_proxy=t_id,
submitted_time=time_submit,
started_time=time_run,
finished_time=time_run_exit,
batch_sys_name=batch_sys_name,
batch_sys_job_id=batch_sys_job_id,
host=j_host,
owner=j_owner,
name=name,
cycle_point=point_string,
)
# Add in log files.
j_buf.job_log_dir = get_task_job_log(self.schd.suite, point_string,
name, submit_num)
overrides = self.schd.task_events_mgr.broadcast_mgr.get_broadcast(
TaskID.get(name, point_string))
if overrides:
rtconfig = pdeepcopy(tdef.rtconfig)
poverride(rtconfig, overrides, prepend=True)
else:
rtconfig = tdef.rtconfig
j_buf.extra_logs.extend([
os.path.expanduser(os.path.expandvars(log_file))
for log_file in rtconfig['extra log files']
])
except SuiteConfigError:
LOG.exception(
('ignoring job %s from the suite run database\n'
'(its task definition has probably been deleted).') % j_id)
except Exception:
LOG.exception('could not load job %s' % j_id)
else:
self.added[j_id] = j_buf
self.task_jobs.setdefault(t_id, set()).add(j_id)
self.updates_pending = True
def test_poverride_prepend(self):
source = OrderedDictWithDefaults()
source["name"] = OrderedDictWithDefaults()
source["name"]["value"] = "oil"
source["name"]["key"] = [1, 2, 3, 4]
target = OrderedDictWithDefaults()
target["name"] = OrderedDictWithDefaults()
target["name"]["index"] = 0
poverride(target, None) # harmless, and no error/exception
poverride(target, source, prepend=True)
expected = OrderedDictWithDefaults()
expected["key"] = [1, 2, 3, 4]
expected["value"] = "oil"
expected["index"] = 0
self.assertEqual(expected, target["name"])
def _prep_submit_task_job(self, suite, itask, dry_run, check_syntax=True):
"""Prepare a task job submission.
Return itask on a good preparation.
"""
if itask.local_job_file_path and not dry_run:
return itask
# Handle broadcasts
overrides = self.task_events_mgr.broadcast_mgr.get_broadcast(
itask.identity)
if overrides:
rtconfig = pdeepcopy(itask.tdef.rtconfig)
poverride(rtconfig, overrides, prepend=True)
else:
rtconfig = itask.tdef.rtconfig
# Determine task host settings now, just before job submission,
# because dynamic host selection may be used.
try:
task_host = self.task_remote_mgr.remote_host_select(
rtconfig['remote']['host'])
except TaskRemoteMgmtError as exc:
# Submit number not yet incremented
itask.submit_num += 1
itask.summary['job_hosts'][itask.submit_num] = ''
# Retry delays, needed for the try_num
self._set_retry_timers(itask, rtconfig)
self._prep_submit_task_job_error(suite, itask, dry_run,
'(remote host select)', exc)
return False
else:
if task_host is None: # host select not ready
itask.set_summary_message(self.REMOTE_SELECT_MSG)
return
itask.task_host = task_host
# Submit number not yet incremented
itask.submit_num += 1
# Retry delays, needed for the try_num
self._set_retry_timers(itask, rtconfig)
try:
job_conf = self._prep_submit_task_job_impl(suite, itask, rtconfig)
local_job_file_path = get_task_job_job_log(suite, itask.point,
itask.tdef.name,
itask.submit_num)
self.job_file_writer.write(local_job_file_path,
job_conf,
check_syntax=check_syntax)
except Exception as exc:
# Could be a bad command template, IOError, etc
self._prep_submit_task_job_error(suite, itask, dry_run,
'(prepare job file)', exc)
return False
itask.local_job_file_path = local_job_file_path
job_config = deepcopy(job_conf)
job_config['logfiles'] = deepcopy(itask.summary['logfiles'])
job_config['job_log_dir'] = get_task_job_log(suite, itask.point,
itask.tdef.name,
itask.submit_num)
itask.jobs.append(job_config['job_d'])
self.job_pool.insert_job(job_config)
if dry_run:
itask.set_summary_message(self.DRY_RUN_MSG)
self.job_pool.add_job_msg(job_config['job_d'], self.DRY_RUN_MSG)
LOG.debug(f'[{itask}] -{self.DRY_RUN_MSG}')
# Return value used by "cylc submit" and "cylc jobscript":
return itask
def _prep_submit_task_job(self, suite, itask, dry_run, check_syntax=True):
"""Prepare a task job submission.
Return itask on a good preparation.
"""
if itask.local_job_file_path and not dry_run:
return itask
# Handle broadcasts
overrides = self.task_events_mgr.broadcast_mgr.get_broadcast(
itask.identity)
if overrides:
rtconfig = pdeepcopy(itask.tdef.rtconfig)
poverride(rtconfig, overrides, prepend=True)
else:
rtconfig = itask.tdef.rtconfig
# Determine task host settings now, just before job submission,
# because dynamic host selection may be used.
try:
task_host = self.task_remote_mgr.remote_host_select(
rtconfig['remote']['host'])
except TaskRemoteMgmtError as exc:
# Submit number not yet incremented
itask.submit_num += 1
itask.summary['job_hosts'][itask.submit_num] = ''
# Retry delays, needed for the try_num
self._set_retry_timers(itask, rtconfig)
self._prep_submit_task_job_error(
suite, itask, dry_run, '(remote host select)', exc)
return False
else:
if task_host is None: # host select not ready
itask.set_summary_message(self.REMOTE_SELECT_MSG)
return
itask.task_host = task_host
# Submit number not yet incremented
itask.submit_num += 1
# Retry delays, needed for the try_num
self._set_retry_timers(itask, rtconfig)
try:
job_conf = self._prep_submit_task_job_impl(suite, itask, rtconfig)
local_job_file_path = get_task_job_job_log(
suite, itask.point, itask.tdef.name, itask.submit_num)
self.job_file_writer.write(local_job_file_path, job_conf,
check_syntax=check_syntax)
except Exception as exc:
# Could be a bad command template, IOError, etc
self._prep_submit_task_job_error(
suite, itask, dry_run, '(prepare job file)', exc)
return False
itask.local_job_file_path = local_job_file_path
if dry_run:
itask.set_summary_message('job file written (edit/dry-run)')
LOG.debug('[%s] -%s', itask, itask.summary['latest_message'])
# Return value used by "cylc submit" and "cylc jobscript":
return itask
def _prep_submit_task_job(self, suite, itask, check_syntax=True):
"""Prepare a task job submission.
Return itask on a good preparation.
"""
if itask.local_job_file_path:
return itask
# Handle broadcasts
overrides = self.task_events_mgr.broadcast_mgr.get_broadcast(
itask.identity)
if overrides:
rtconfig = pdeepcopy(itask.tdef.rtconfig)
poverride(rtconfig, overrides, prepend=True)
else:
rtconfig = itask.tdef.rtconfig
# TODO - remove host logic at Cylc 9
# Determine task host or platform now, just before job submission,
# because dynamic host/platform selection may be used.
# cases:
# - Platform exists, host does = throw error here:
# Although errors of this sort should ideally be caught on config
# load this cannot be done because inheritance may create conflicts
# which appear later. Although this error is also raised
# by the platforms module it's probably worth putting it here too
# to prevent trying to run the remote_host/platform_select logic for
# tasks which will fail anyway later.
# - Platform exists, host doesn't = eval platform_n
# - host exists - eval host_n
if (rtconfig['platform'] is not None
and rtconfig['remote']['host'] is not None):
raise SuiteConfigError(
"A mixture of Cylc 7 (host) and Cylc 8 (platform) "
"logic should not be used. In this case for the task "
f"\"{itask.identity}\" the following are not compatible:\n")
host_n, platform_n = None, None
try:
if rtconfig['remote']['host'] is not None:
host_n = self.task_remote_mgr.subshell_eval(
rtconfig['remote']['host'], HOST_REC_COMMAND)
else:
platform_n = self.task_remote_mgr.subshell_eval(
rtconfig['platform'], PLATFORM_REC_COMMAND)
except TaskRemoteMgmtError as exc:
# Submit number not yet incremented
itask.submit_num += 1
itask.summary['platforms_used'][itask.submit_num] = ''
# Retry delays, needed for the try_num
self._create_job_log_path(suite, itask)
self._set_retry_timers(itask, rtconfig)
self._prep_submit_task_job_error(suite, itask,
'(remote host select)', exc)
return False
else:
# host/platform select not ready
if host_n is None and platform_n is None:
itask.set_summary_message(self.REMOTE_SELECT_MSG)
return
elif host_n is None and rtconfig['platform'] != platform_n:
LOG.debug(f"for task {itask.identity}: platform = "
f"{rtconfig['platform']} evaluated as {platform_n}")
rtconfig['platform'] = platform_n
elif platform_n is None and rtconfig['remote']['host'] != host_n:
LOG.debug(
f"for task {itask.identity}: host = "
f"{rtconfig['remote']['host']} evaluated as {host_n}")
rtconfig['remote']['host'] = host_n
try:
platform = get_platform(rtconfig)
except PlatformLookupError as exc:
# Submit number not yet incremented
itask.submit_num += 1
itask.summary['platforms_used'][itask.submit_num] = ''
# Retry delays, needed for the try_num
self._create_job_log_path(suite, itask)
self._set_retry_timers(itask, rtconfig, False)
self._prep_submit_task_job_error(suite, itask,
'(platform not defined)', exc)
return False
else:
itask.platform = platform
# Submit number not yet incremented
itask.submit_num += 1
# Retry delays, needed for the try_num
self._set_retry_timers(itask, rtconfig)
try:
job_conf = self._prep_submit_task_job_impl(suite, itask, rtconfig)
# Job pool insertion
job_config = deepcopy(job_conf)
job_config['logfiles'] = deepcopy(itask.summary['logfiles'])
itask.jobs.append(job_config['job_d'])
self.job_pool.insert_job(job_config)
local_job_file_path = get_task_job_job_log(suite, itask.point,
itask.tdef.name,
itask.submit_num)
self.job_file_writer.write(local_job_file_path,
job_conf,
check_syntax=check_syntax)
except Exception as exc:
# Could be a bad command template, IOError, etc
self._prep_submit_task_job_error(suite, itask,
'(prepare job file)', exc)
return False
itask.local_job_file_path = local_job_file_path
return itask