def run(log, g_vars, qaOpts, rawCfgPars, cfgFile): #g_vars.TTY = os.ttyname(0) # are git and wget available? g_vars.NO_GIT=True if qa_util.which('git'): g_vars.NO_GIT = False g_vars.NO_WGET=True if qa_util.which('wget'): g_vars.NO_WGET = False # update external tables and in case of running qa_dkrz.py from # sources update C++ executables run_install(qaOpts, g_vars) if qaOpts.isOpt('NUM_EXEC_THREADS'): g_vars.thread_num = \ sum( qa_util.mk_list(qaOpts.getOpt('NUM_EXEC_THREADS')) ) else: g_vars.thread_num = 1 g_vars.res_dir_path = qaOpts.getOpt('QA_RESULTS') g_vars.project_data_path = qaOpts.getOpt('PROJECT_DATA') g_vars.prj_dp_len = len(g_vars.project_data_path) init_session(g_vars, qaOpts) g_vars.check_logs_path = os.path.join(g_vars.res_dir_path, 'check_logs') g_vars.cs_enable = False if qaOpts.isOpt('CHECKSUM'): g_vars.cs_enable = True if qaOpts.isOpt('CHECKSUM', True): g_vars.cs_type = 'md5' else: g_vars.cs_type = qaOpts.getOpt('CHECKSUM') cs_dir = qaOpts.getOpt('CS_DIR') if len(cs_dir) == 0: cs_dir='cs_table' g_vars.cs_dir = os.path.join(g_vars.res_dir_path, cs_dir) qaOpts.setOpt('LOG_FNAME_DIR', g_vars.check_logs_path) qa_util.mkdirP(g_vars.check_logs_path) # error --> exit qa_util.mkdirP(os.path.join(g_vars.res_dir_path, 'data')) # error --> exit # some more settings if not qaOpts.isOpt('ZOMBIE_LIMIT'): qaOpts.setOpt('ZOMBIE_LIMIT', 3600) if not qaOpts.isOpt('CHECKSUM'): if qaOpts.isOpt('CS_STAND_ALONE') or qaOpts.isOpt('CS_DIR'): qaOpts.setOpt('CHECKSUM', True) # save current version id to the cfg-file qv=qaOpts.getOpt('QA_REVISION') if len(qv) == 0: qv=qa_util.get_curr_revision(g_vars.qa_src, g_vars.isConda) qv = qa_util.cfg_parser(rawCfgPars, cfgFile, section=g_vars.qa_src, key='QA_REVISION', value=qv) qaOpts.setOpt('QA_REVISION', qv) g_vars.qa_revision = qv # table path and copy of tables for operational runs init_tables(g_vars, qaOpts) # unique exp_name and table_names are defined by indices of path components qa_util.get_experiment_name(g_vars, qaOpts, isInit=True) qa_util.get_project_table_name(g_vars, qaOpts, isInit=True) # enable clearance of logfile entries by the CLEAR option if qaOpts.isOpt('CLEAR_LOGFILE'): g_vars.clear_logfile = True else: g_vars.clear_logfile = False g_vars.ignore_temp_files = qaOpts.isOpt('IGNORE_TEMP_FILES') g_vars.syncFilePrg = os.path.join(g_vars.qa_src, 'bin', 'syncFiles.x') g_vars.checkPrg = os.path.join(g_vars.qa_src, 'bin', 'qA-' + qaOpts.getOpt('PROJECT_AS') + '.x') if not os.access(g_vars.syncFilePrg, os.X_OK): print g_vars.syncFilePrg + ' is not executable' sys.exit(1) if not os.access(g_vars.checkPrg, os.X_OK): print g_vars.checkPrg + ' is not executable' sys.exit(1) g_vars.anyProgress = False return
def prepareExample(qaConf): if not qaConf.isOpt('PROJECT'): qaConf.addOpt("PROJECT", 'CORDEX') if qaConf.isOpt('WORK'): currdir = qaConf.getOpt('WORK') else: currdir = qaConf.getOpt('CURR_DIR') #currdir=os.path.join(currdir, 'example') qaConf.dOpts['QA_RESULTS'] = os.path.join(currdir, 'results') if not qa_util.mkdirP(currdir): sys.exit(1) os.chdir(currdir) qa_util.rm_r('results', 'config.txt', 'qa-test.task') print 'make examples in ' + currdir print 'make qa_test.task' taskFile = os.path.join(QA_SRC, 'example', 'templates', 'qa-test.task') shutil.copy(taskFile, currdir) taskFile = 'qa-test.task' # replace templates within QA_SRC/example sub = [] repl = [] sub.append('PROJECT_DATA=data') repl.append('PROJECT_DATA=' + os.path.join(currdir, 'data')) sub.append('QA_RESULTS=results') repl.append('QA_RESULTS=' + os.path.join(currdir, 'results')) qa_util.f_str_replace(taskFile, sub, repl) if not qa_util.which("ncgen"): print "building data in example requires the ncgen utility" sys.exit(1) if not os.path.isdir(os.path.join(currdir, 'data')): # data print 'make data' p = os.path.join(QA_SRC, 'example', 'templates', 'data.tbz') subprocess.call(["tar", "--bzip2", "-xf", p]) for rs, ds, fs in os.walk('data'): for f in fs: nc_f = f[:len(f) - 3] + 'nc' t_f = os.path.join(rs, f) t_nc = os.path.join(rs, nc_f) try: subprocess.call(["ncgen", "-k", "3", "-o", t_nc, t_f]) except: print 'making of example failed' sys.exit(1) else: qa_util.rm_r(t_f) print 'run' + sys.argv[0] + " -m -f " + currdir + "/qa-test.task" qaConf = QaConfig(QA_SRC, ['-m', '-f', currdir + "/qa-test.task"]) return qaConf
def runExample(): if qaOpts.isOpt('EXAMPLE_PATH'): currdir = os.path.join( qaOpts.getOpt('EXAMPLE_PATH'), 'example') if currdir[0] != '/': currdir = os.path.join(os.getcwd(), currdir) else: currdir=os.path.join(QA_SRC, 'example') if not qa_util.mkdirP(currdir): print 'could not mkdir ' + dir + ', please use option --example=path' sys.exit(1) os.chdir(currdir) qa_util.rmR( 'results', 'config.txt', 'data', 'tables', 'qa-test.task' ) print 'make examples in ' + currdir print 'make qa_test.task' taskFile = os.path.join(QA_SRC, 'example', 'templates', 'qa-test.task') shutil.copy( taskFile, currdir) taskFile = 'qa-test.task' # replace templates sub=[] repl=[] sub.append('PROJECT_DATA=data') repl.append('PROJECT_DATA=' + os.path.join(currdir, 'data') ) sub.append('QA_RESULTS=results') repl.append('QA_RESULTS=' + os.path.join(currdir, 'results') ) qa_util.f_str_replace(taskFile, sub, repl) # data print 'make data' subprocess.call(["tar", "--bzip2", "-xf", \ os.path.join(QA_SRC,os.path.join('example', 'templates', 'data.tbz') ) ]) txtFs=[] for rs, ds, fs in os.walk('data'): for f in fs: if '.txt' in f: txtFs.append(os.path.join(rs,f)) if qa_util.which("ncgen"): for f in txtFs: nc_f = f[:len(f)-2] + 'nc' subprocess.call(["ncgen", "-k", "3", "-o", nc_f, f]) qa_util.rmR(f) else: print "building data in example requires the ncgen utility" print 'run' print os.path.join(QA_SRC, 'scripts', 'qa-dkrz') +\ " -m --work=" + currdir + '-f qa-test.task' subprocess.call([os.path.join(QA_SRC, 'scripts', 'qa-dkrz'), \ '--work=' + currdir, "-f", "qa-test.task"]) return