def final(): # only the summary of previous runs if not qaOpts.isOpt('NO_SUMMARY') and not qaOpts.isOpt('SHOW'): summary() # remove duplicates for log_fname in g_vars.log_fnames: tmp_log = os.path.join(g_vars.check_logs_path, 'tmp_' + log_fname + '.log') dest_log = os.path.join(g_vars.check_logs_path, log_fname + '.log') if not os.path.isfile(tmp_log): continue # nothing new if os.path.isfile(dest_log): if qaOpts.isOpt('CLEAR_LOGFILE'): ix = g_vars.log_fnames.index(log_fname) fBase = g_vars.clear_fBase[ix] clrdName='cleared_' + log_fname + '.log' clrdFile=os.path.join(g_vars.check_logs_path, clrdName) with open(clrdFile, 'w') as clrd_fd: while True: blk = log.get_next_blk(dest_log, skip_fBase=fBase, skip_prmbl=False) for b in blk: clrd_fd.write(b) else: break if clrd_fd.errors == None: os.rename(clrdFile, dest_log) # append recent results to a logfile qa_util.cat(tmp_log, dest_log, append=True) os.remove(tmp_log) else: # first time that a check was done for this log-file os.rename(tmp_log, dest_log) qa_util.cfg_parser(rawCfgPars, qaOpts.getOpt('CFG_FILE'), final=True) return
def run(log, g_vars, qaOpts, rawCfgPars, cfgFile): #g_vars.TTY = os.ttyname(0) # are git and wget available? g_vars.NO_GIT=True if qa_util.which('git'): g_vars.NO_GIT = False g_vars.NO_WGET=True if qa_util.which('wget'): g_vars.NO_WGET = False # update external tables and in case of running qa_dkrz.py from # sources update C++ executables run_install(qaOpts, g_vars) if qaOpts.isOpt('NUM_EXEC_THREADS'): g_vars.thread_num = \ sum( qa_util.mk_list(qaOpts.getOpt('NUM_EXEC_THREADS')) ) else: g_vars.thread_num = 1 g_vars.res_dir_path = qaOpts.getOpt('QA_RESULTS') g_vars.project_data_path = qaOpts.getOpt('PROJECT_DATA') g_vars.prj_dp_len = len(g_vars.project_data_path) init_session(g_vars, qaOpts) g_vars.check_logs_path = os.path.join(g_vars.res_dir_path, 'check_logs') g_vars.cs_enable = False if qaOpts.isOpt('CHECKSUM'): g_vars.cs_enable = True if qaOpts.isOpt('CHECKSUM', True): g_vars.cs_type = 'md5' else: g_vars.cs_type = qaOpts.getOpt('CHECKSUM') cs_dir = qaOpts.getOpt('CS_DIR') if len(cs_dir) == 0: cs_dir='cs_table' g_vars.cs_dir = os.path.join(g_vars.res_dir_path, cs_dir) qaOpts.setOpt('LOG_FNAME_DIR', g_vars.check_logs_path) qa_util.mkdirP(g_vars.check_logs_path) # error --> exit qa_util.mkdirP(os.path.join(g_vars.res_dir_path, 'data')) # error --> exit # some more settings if not qaOpts.isOpt('ZOMBIE_LIMIT'): qaOpts.setOpt('ZOMBIE_LIMIT', 3600) if not qaOpts.isOpt('CHECKSUM'): if qaOpts.isOpt('CS_STAND_ALONE') or qaOpts.isOpt('CS_DIR'): qaOpts.setOpt('CHECKSUM', True) # save current version id to the cfg-file qv=qaOpts.getOpt('QA_REVISION') if len(qv) == 0: qv=qa_util.get_curr_revision(g_vars.qa_src, g_vars.isConda) qv = qa_util.cfg_parser(rawCfgPars, cfgFile, section=g_vars.qa_src, key='QA_REVISION', value=qv) qaOpts.setOpt('QA_REVISION', qv) g_vars.qa_revision = qv # table path and copy of tables for operational runs init_tables(g_vars, qaOpts) # unique exp_name and table_names are defined by indices of path components qa_util.get_experiment_name(g_vars, qaOpts, isInit=True) qa_util.get_project_table_name(g_vars, qaOpts, isInit=True) # enable clearance of logfile entries by the CLEAR option if qaOpts.isOpt('CLEAR_LOGFILE'): g_vars.clear_logfile = True else: g_vars.clear_logfile = False g_vars.ignore_temp_files = qaOpts.isOpt('IGNORE_TEMP_FILES') g_vars.syncFilePrg = os.path.join(g_vars.qa_src, 'bin', 'syncFiles.x') g_vars.checkPrg = os.path.join(g_vars.qa_src, 'bin', 'qA-' + qaOpts.getOpt('PROJECT_AS') + '.x') if not os.access(g_vars.syncFilePrg, os.X_OK): print g_vars.syncFilePrg + ' is not executable' sys.exit(1) if not os.access(g_vars.checkPrg, os.X_OK): print g_vars.checkPrg + ' is not executable' sys.exit(1) g_vars.anyProgress = False return
if clearInq(t_vars.var_path, fBase, logfile): return True # any qa_lock file? f_lock = os.path.join(t_vars.var_path, 'qa_lock_' + fBase + '.txt') if os.path.exists(f_lock): return True return False # -------- main -------- # read (may-be convert) the cfg files qa_util.cfg_parser(rawCfgPars, qaOpts.getOpt('CFG_FILE'), init=True) if 'QA_EXAMPLE' in qaOpts.dOpts: runExample() sys.exit(0) qa_init.run(log, g_vars, qaOpts, rawCfgPars, qaOpts.getOpt('CFG_FILE')) # the checks if not qaOpts.getOpt('ONLY_SUMMARY'): run() final() if __name__ == '__main__':