def resubmit_check(case):
###############################################################################
# check to see if we need to do resubmission from this particular job,
# Note that Mira requires special logic
dout_s = case.get_value("DOUT_S")
logger.warn("dout_s %s "%(dout_s))
mach = case.get_value("MACH")
logger.warn("mach %s "%(mach))
testcase = case.get_value("TESTCASE")
resubmit_num = case.get_value("RESUBMIT")
logger.warn("resubmit_num %s"%(resubmit_num))
# If dout_s is True than short-term archiving handles the resubmit
# If dout_s is True and machine is mira submit the st_archive script
resubmit = False
if not dout_s and resubmit_num > 0:
resubmit = True
elif dout_s and mach == 'mira':
caseroot = case.get_value("CASEROOT")
cimeroot = case.get_value("CIMEROOT")
cmd = "ssh cooleylogin1 'cd %s; CIMEROOT=%s ./case.submit %s --job case.st_archive' "%(caseroot, cimeroot, caseroot)
run_cmd(cmd, verbose=True)
if resubmit:
if testcase is not None and testcase in ['ERR']:
job = "case.test"
else:
job = "case.run"
submit(case, job=job, resubmit=True)
def case_st_archive(case):
###############################################################################
caseroot = case.get_value("CASEROOT")
logger.info("st_archive starting")
# do short-term archiving
append_status("st_archiving starting",
caseroot=caseroot, sfile="CaseStatus")
cmd = os.path.join(caseroot, "Tools/st_archive") + " >> stArchiveStatus 2>&1"
rc, out, err = run_cmd(cmd, ok_to_fail=True)
if rc != 0:
append_status("st_archive failed: %s \nerr = %s"%(out,err),sfile="CaseStatus")
return False
append_status("st_archiving completed",
caseroot=caseroot, sfile="CaseStatus")
logger.info("st_archive completed")
# resubmit case if appropriate
resubmit = case.get_value("RESUBMIT")
if resubmit > 0:
append_status("resubmitting from st_archive",
caseroot=caseroot, sfile="CaseStatus")
logger.info("resubmitting from st_archive, resubmit=%d"%resubmit)
submit(case, resubmit=True)
return True
def resubmitCheck(case):
###############################################################################
# check to see if we need to do resubmission from this particular job,
# Note that Mira requires special logic
dout_s = case.get_value("DOUT_S")
mach = case.get_value("MACH")
testcase = case.get_value("TESTCASE")
resubmit_num = case.get_value("RESUBMIT")
# If dout_s is True than short-term archiving handles the resubmit
# that is not the case on Mira
resubmit = False
if not dout_s and resubmit_num > 0:
resubmit = True
elif dout_s and mach == 'mira':
resubmit = True
if resubmit:
if testcase is not None and testcase in ['ERR']:
job = "case.test"
else:
job = "case.run"
submit(case, job=job, resubmit=True)
def case_st_archive(case, last_date_str=None, archive_incomplete_logs=True, copy_only=False, no_resubmit=False):
###############################################################################
"""
Create archive object and perform short term archiving
"""
caseroot = case.get_value("CASEROOT")
if last_date_str is not None:
try:
last_date = datetime.datetime.strptime(last_date_str, '%Y-%m-%d')
except ValueError:
expect(False, 'Could not parse the last date to archive')
else:
last_date = None
dout_s_root = case.get_value('DOUT_S_ROOT')
if dout_s_root is None or dout_s_root == 'UNSET':
expect(False,
'XML variable DOUT_S_ROOT is required for short-term achiver')
if not isdir(dout_s_root):
os.makedirs(dout_s_root)
dout_s_save_interim = case.get_value('DOUT_S_SAVE_INTERIM_RESTART_FILES')
if dout_s_save_interim == 'FALSE' or dout_s_save_interim == 'UNSET':
rest_n = case.get_value('REST_N')
stop_n = case.get_value('STOP_N')
if rest_n < stop_n:
logger.warning('Restart files from end of run will be saved'
'interim restart files will be deleted')
logger.info("st_archive starting")
archive = case.get_env('archive')
functor = lambda: _archive_process(case, archive, last_date, archive_incomplete_logs, copy_only)
run_and_log_case_status(functor, "st_archive", caseroot=caseroot)
logger.info("st_archive completed")
# resubmit case if appropriate
resubmit = case.get_value("RESUBMIT")
if resubmit > 0 and not no_resubmit:
logger.info("resubmitting from st_archive, resubmit={:d}".format(resubmit))
if case.get_value("MACH") == "mira":
expect(os.path.isfile(".original_host"), "ERROR alcf host file not found")
with open(".original_host", "r") as fd:
sshhost = fd.read()
run_cmd("ssh cooleylogin1 ssh {} '{}/case.submit {} --resubmit' "\
.format(sshhost, caseroot, caseroot), verbose=True)
else:
submit(case, resubmit=True)
return True
def case_st_archive(case):
###############################################################################
"""
Create archive object and perform short term archiving
"""
caseroot = case.get_value("CASEROOT")
dout_s_root = case.get_value("DOUT_S_ROOT")
if dout_s_root is None or dout_s_root == "UNSET":
expect(False, "XML variable DOUT_S_ROOT is required for short-term achiver")
if not isdir(dout_s_root):
os.makedirs(dout_s_root)
dout_s_save_interim = case.get_value("DOUT_S_SAVE_INTERIM_RESTART_FILES")
if dout_s_save_interim == "FALSE" or dout_s_save_interim == "UNSET":
rest_n = case.get_value("REST_N")
stop_n = case.get_value("STOP_N")
if rest_n < stop_n:
logger.warn("Restart files from end of run will be saved" "interim restart files will be deleted")
logger.info("st_archive starting")
# do short-term archiving
append_status("st_archiving starting", caseroot=caseroot, sfile="CaseStatus")
archive = EnvArchive(infile=os.path.join(caseroot, "env_archive.xml"))
_archive_process(case, archive)
append_status("st_archiving completed", caseroot=caseroot, sfile="CaseStatus")
logger.info("st_archive completed")
# resubmit case if appropriate
resubmit = case.get_value("RESUBMIT")
if resubmit > 0:
append_status("resubmitting from st_archive", caseroot=caseroot, sfile="CaseStatus")
logger.info("resubmitting from st_archive, resubmit=%d" % resubmit)
if case.get_value("MACH") == "mira":
expect(os.path.isfile(".original_host"), "ERROR alcf host file not found")
with open(".original_host", "r") as fd:
sshhost = fd.read()
run_cmd(
"ssh cooleylogin1 ssh %s '%s/case.submit %s --resubmit' " % (sshhost, caseroot, caseroot), verbose=True
)
else:
submit(case, resubmit=True)
return True