def final():
# only the summary of previous runs
if not qaOpts.isOpt('NO_SUMMARY') and not qaOpts.isOpt('SHOW'):
summary()
# remove duplicates
for log_fname in g_vars.log_fnames:
tmp_log = os.path.join(g_vars.check_logs_path,
'tmp_' + log_fname + '.log')
dest_log = os.path.join(g_vars.check_logs_path,
log_fname + '.log')
if not os.path.isfile(tmp_log):
continue # nothing new
if os.path.isfile(dest_log):
if qaOpts.isOpt('CLEAR_LOGFILE'):
ix = g_vars.log_fnames.index(log_fname)
fBase = g_vars.clear_fBase[ix]
clrdName='cleared_' + log_fname + '.log'
clrdFile=os.path.join(g_vars.check_logs_path, clrdName)
with open(clrdFile, 'w') as clrd_fd:
while True:
blk = log.get_next_blk(dest_log,
skip_fBase=fBase,
skip_prmbl=False)
for b in blk:
clrd_fd.write(b)
else:
break
if clrd_fd.errors == None:
os.rename(clrdFile, dest_log)
# append recent results to a logfile
qa_util.cat(tmp_log, dest_log, append=True)
os.remove(tmp_log)
else:
# first time that a check was done for this log-file
os.rename(tmp_log, dest_log)
qa_util.cfg_parser(rawCfgPars, qaOpts.getOpt('CFG_FILE'), final=True)
return
def run(log, g_vars, qaOpts, rawCfgPars, cfgFile):
#g_vars.TTY = os.ttyname(0)
# are git and wget available?
g_vars.NO_GIT=True
if qa_util.which('git'):
g_vars.NO_GIT = False
g_vars.NO_WGET=True
if qa_util.which('wget'):
g_vars.NO_WGET = False
# update external tables and in case of running qa_dkrz.py from
# sources update C++ executables
run_install(qaOpts, g_vars)
if qaOpts.isOpt('NUM_EXEC_THREADS'):
g_vars.thread_num = \
sum( qa_util.mk_list(qaOpts.getOpt('NUM_EXEC_THREADS')) )
else:
g_vars.thread_num = 1
g_vars.res_dir_path = qaOpts.getOpt('QA_RESULTS')
g_vars.project_data_path = qaOpts.getOpt('PROJECT_DATA')
g_vars.prj_dp_len = len(g_vars.project_data_path)
init_session(g_vars, qaOpts)
g_vars.check_logs_path = os.path.join(g_vars.res_dir_path, 'check_logs')
g_vars.cs_enable = False
if qaOpts.isOpt('CHECKSUM'):
g_vars.cs_enable = True
if qaOpts.isOpt('CHECKSUM', True):
g_vars.cs_type = 'md5'
else:
g_vars.cs_type = qaOpts.getOpt('CHECKSUM')
cs_dir = qaOpts.getOpt('CS_DIR')
if len(cs_dir) == 0:
cs_dir='cs_table'
g_vars.cs_dir = os.path.join(g_vars.res_dir_path, cs_dir)
qaOpts.setOpt('LOG_FNAME_DIR', g_vars.check_logs_path)
qa_util.mkdirP(g_vars.check_logs_path) # error --> exit
qa_util.mkdirP(os.path.join(g_vars.res_dir_path, 'data')) # error --> exit
# some more settings
if not qaOpts.isOpt('ZOMBIE_LIMIT'):
qaOpts.setOpt('ZOMBIE_LIMIT', 3600)
if not qaOpts.isOpt('CHECKSUM'):
if qaOpts.isOpt('CS_STAND_ALONE') or qaOpts.isOpt('CS_DIR'):
qaOpts.setOpt('CHECKSUM', True)
# save current version id to the cfg-file
qv=qaOpts.getOpt('QA_REVISION')
if len(qv) == 0:
qv=qa_util.get_curr_revision(g_vars.qa_src, g_vars.isConda)
qv = qa_util.cfg_parser(rawCfgPars, cfgFile,
section=g_vars.qa_src, key='QA_REVISION', value=qv)
qaOpts.setOpt('QA_REVISION', qv)
g_vars.qa_revision = qv
# table path and copy of tables for operational runs
init_tables(g_vars, qaOpts)
# unique exp_name and table_names are defined by indices of path components
qa_util.get_experiment_name(g_vars, qaOpts, isInit=True)
qa_util.get_project_table_name(g_vars, qaOpts, isInit=True)
# enable clearance of logfile entries by the CLEAR option
if qaOpts.isOpt('CLEAR_LOGFILE'):
g_vars.clear_logfile = True
else:
g_vars.clear_logfile = False
g_vars.ignore_temp_files = qaOpts.isOpt('IGNORE_TEMP_FILES')
g_vars.syncFilePrg = os.path.join(g_vars.qa_src, 'bin', 'syncFiles.x')
g_vars.checkPrg = os.path.join(g_vars.qa_src, 'bin',
'qA-' + qaOpts.getOpt('PROJECT_AS') + '.x')
if not os.access(g_vars.syncFilePrg, os.X_OK):
print g_vars.syncFilePrg + ' is not executable'
sys.exit(1)
if not os.access(g_vars.checkPrg, os.X_OK):
print g_vars.checkPrg + ' is not executable'
sys.exit(1)
g_vars.anyProgress = False
return
if clearInq(t_vars.var_path, fBase, logfile):
return True
# any qa_lock file?
f_lock = os.path.join(t_vars.var_path, 'qa_lock_' + fBase + '.txt')
if os.path.exists(f_lock):
return True
return False
# -------- main --------
# read (may-be convert) the cfg files
qa_util.cfg_parser(rawCfgPars, qaOpts.getOpt('CFG_FILE'), init=True)
if 'QA_EXAMPLE' in qaOpts.dOpts:
runExample()
sys.exit(0)
qa_init.run(log, g_vars, qaOpts, rawCfgPars,
qaOpts.getOpt('CFG_FILE'))
# the checks
if not qaOpts.getOpt('ONLY_SUMMARY'):
run()
final()
if __name__ == '__main__':
