Пример #1
0
def final():

    # only the summary of previous runs
    if not qaOpts.isOpt('NO_SUMMARY') and not qaOpts.isOpt('SHOW'):
        summary()

    # remove duplicates
    for log_fname in g_vars.log_fnames:
        tmp_log = os.path.join(g_vars.check_logs_path,
                               'tmp_' + log_fname + '.log')
        dest_log = os.path.join(g_vars.check_logs_path,
                                log_fname + '.log')

        if not os.path.isfile(tmp_log):
            continue  # nothing new

        if os.path.isfile(dest_log):
            if qaOpts.isOpt('CLEAR_LOGFILE'):
                ix = g_vars.log_fnames.index(log_fname)
                fBase = g_vars.clear_fBase[ix]

                clrdName='cleared_' + log_fname + '.log'
                clrdFile=os.path.join(g_vars.check_logs_path, clrdName)

                with open(clrdFile, 'w') as clrd_fd:
                    while True:
                        blk = log.get_next_blk(dest_log,
                                               skip_fBase=fBase,
                                               skip_prmbl=False)

                        for b in blk:
                            clrd_fd.write(b)
                        else:
                            break

                if clrd_fd.errors == None:
                    os.rename(clrdFile, dest_log)

            # append recent results to a logfile
            qa_util.cat(tmp_log, dest_log, append=True)
            os.remove(tmp_log)

        else:
            # first time that a check was done for this log-file
            os.rename(tmp_log, dest_log)

    qa_util.cfg_parser(rawCfgPars, qaOpts.getOpt('CFG_FILE'), final=True)

    return
Пример #2
0
def run(log, g_vars, qaOpts, rawCfgPars, cfgFile):
    #g_vars.TTY = os.ttyname(0)

    # are git and wget available?
    g_vars.NO_GIT=True
    if qa_util.which('git'):
        g_vars.NO_GIT = False

    g_vars.NO_WGET=True
    if qa_util.which('wget'):
        g_vars.NO_WGET = False

    # update external tables and in case of running qa_dkrz.py from
    # sources update C++ executables
    run_install(qaOpts, g_vars)

    if qaOpts.isOpt('NUM_EXEC_THREADS'):
        g_vars.thread_num = \
            sum( qa_util.mk_list(qaOpts.getOpt('NUM_EXEC_THREADS')) )
    else:
        g_vars.thread_num = 1

    g_vars.res_dir_path = qaOpts.getOpt('QA_RESULTS')
    g_vars.project_data_path = qaOpts.getOpt('PROJECT_DATA')
    g_vars.prj_dp_len = len(g_vars.project_data_path)

    init_session(g_vars, qaOpts)

    g_vars.check_logs_path = os.path.join(g_vars.res_dir_path, 'check_logs')

    g_vars.cs_enable = False
    if qaOpts.isOpt('CHECKSUM'):
        g_vars.cs_enable = True
        if qaOpts.isOpt('CHECKSUM', True):
            g_vars.cs_type = 'md5'
        else:
            g_vars.cs_type = qaOpts.getOpt('CHECKSUM')

        cs_dir = qaOpts.getOpt('CS_DIR')
        if len(cs_dir) == 0:
            cs_dir='cs_table'
        g_vars.cs_dir = os.path.join(g_vars.res_dir_path, cs_dir)

    qaOpts.setOpt('LOG_FNAME_DIR', g_vars.check_logs_path)
    qa_util.mkdirP(g_vars.check_logs_path) # error --> exit

    qa_util.mkdirP(os.path.join(g_vars.res_dir_path, 'data')) # error --> exit

    # some more settings
    if not qaOpts.isOpt('ZOMBIE_LIMIT'):
        qaOpts.setOpt('ZOMBIE_LIMIT', 3600)

    if not qaOpts.isOpt('CHECKSUM'):
        if qaOpts.isOpt('CS_STAND_ALONE') or qaOpts.isOpt('CS_DIR'):
            qaOpts.setOpt('CHECKSUM', True)

    # save current version id to the cfg-file
    qv=qaOpts.getOpt('QA_REVISION')
    if len(qv) == 0:
        qv=qa_util.get_curr_revision(g_vars.qa_src, g_vars.isConda)

    qv = qa_util.cfg_parser(rawCfgPars, cfgFile,
                           section=g_vars.qa_src, key='QA_REVISION', value=qv)

    qaOpts.setOpt('QA_REVISION', qv)
    g_vars.qa_revision = qv

    # table path and copy of tables for operational runs
    init_tables(g_vars, qaOpts)

    # unique exp_name and table_names are defined by indices of path components
    qa_util.get_experiment_name(g_vars, qaOpts, isInit=True)
    qa_util.get_project_table_name(g_vars, qaOpts, isInit=True)

    # enable clearance of logfile entries by the CLEAR option
    if qaOpts.isOpt('CLEAR_LOGFILE'):
        g_vars.clear_logfile = True
    else:
        g_vars.clear_logfile = False

    g_vars.ignore_temp_files = qaOpts.isOpt('IGNORE_TEMP_FILES')
    g_vars.syncFilePrg = os.path.join(g_vars.qa_src, 'bin', 'syncFiles.x')
    g_vars.checkPrg = os.path.join(g_vars.qa_src, 'bin',
                                     'qA-' + qaOpts.getOpt('PROJECT_AS') + '.x')

    if not os.access(g_vars.syncFilePrg, os.X_OK):
        print g_vars.syncFilePrg + ' is not executable'
        sys.exit(1)
    if not os.access(g_vars.checkPrg, os.X_OK):
        print g_vars.checkPrg + ' is not executable'
        sys.exit(1)

    g_vars.anyProgress = False

    return
Пример #3
0
        if clearInq(t_vars.var_path, fBase, logfile):
            return True

    # any qa_lock file?
    f_lock = os.path.join(t_vars.var_path, 'qa_lock_' + fBase + '.txt')
    if os.path.exists(f_lock):
        return True

    return False



# -------- main --------
# read (may-be convert) the cfg files
qa_util.cfg_parser(rawCfgPars, qaOpts.getOpt('CFG_FILE'), init=True)

if 'QA_EXAMPLE' in qaOpts.dOpts:
    runExample()
    sys.exit(0)

qa_init.run(log, g_vars, qaOpts, rawCfgPars,
            qaOpts.getOpt('CFG_FILE'))

# the checks
if not qaOpts.getOpt('ONLY_SUMMARY'):
    run()

final()

if __name__ == '__main__':