def run(log, g_vars, qaOpts, rawCfgPars, cfgFile):
#g_vars.TTY = os.ttyname(0)
# are git and wget available?
g_vars.NO_GIT=True
if qa_util.which('git'):
g_vars.NO_GIT = False
g_vars.NO_WGET=True
if qa_util.which('wget'):
g_vars.NO_WGET = False
# update external tables and in case of running qa_dkrz.py from
# sources update C++ executables
run_install(qaOpts, g_vars)
if qaOpts.isOpt('NUM_EXEC_THREADS'):
g_vars.thread_num = \
sum( qa_util.mk_list(qaOpts.getOpt('NUM_EXEC_THREADS')) )
else:
g_vars.thread_num = 1
g_vars.res_dir_path = qaOpts.getOpt('QA_RESULTS')
g_vars.project_data_path = qaOpts.getOpt('PROJECT_DATA')
g_vars.prj_dp_len = len(g_vars.project_data_path)
init_session(g_vars, qaOpts)
g_vars.check_logs_path = os.path.join(g_vars.res_dir_path, 'check_logs')
g_vars.cs_enable = False
if qaOpts.isOpt('CHECKSUM'):
g_vars.cs_enable = True
if qaOpts.isOpt('CHECKSUM', True):
g_vars.cs_type = 'md5'
else:
g_vars.cs_type = qaOpts.getOpt('CHECKSUM')
cs_dir = qaOpts.getOpt('CS_DIR')
if len(cs_dir) == 0:
cs_dir='cs_table'
g_vars.cs_dir = os.path.join(g_vars.res_dir_path, cs_dir)
qaOpts.setOpt('LOG_FNAME_DIR', g_vars.check_logs_path)
qa_util.mkdirP(g_vars.check_logs_path) # error --> exit
qa_util.mkdirP(os.path.join(g_vars.res_dir_path, 'data')) # error --> exit
# some more settings
if not qaOpts.isOpt('ZOMBIE_LIMIT'):
qaOpts.setOpt('ZOMBIE_LIMIT', 3600)
if not qaOpts.isOpt('CHECKSUM'):
if qaOpts.isOpt('CS_STAND_ALONE') or qaOpts.isOpt('CS_DIR'):
qaOpts.setOpt('CHECKSUM', True)
# save current version id to the cfg-file
qv=qaOpts.getOpt('QA_REVISION')
if len(qv) == 0:
qv=qa_util.get_curr_revision(g_vars.qa_src, g_vars.isConda)
qv = qa_util.cfg_parser(rawCfgPars, cfgFile,
section=g_vars.qa_src, key='QA_REVISION', value=qv)
qaOpts.setOpt('QA_REVISION', qv)
g_vars.qa_revision = qv
# table path and copy of tables for operational runs
init_tables(g_vars, qaOpts)
# unique exp_name and table_names are defined by indices of path components
qa_util.get_experiment_name(g_vars, qaOpts, isInit=True)
qa_util.get_project_table_name(g_vars, qaOpts, isInit=True)
# enable clearance of logfile entries by the CLEAR option
if qaOpts.isOpt('CLEAR_LOGFILE'):
g_vars.clear_logfile = True
else:
g_vars.clear_logfile = False
g_vars.ignore_temp_files = qaOpts.isOpt('IGNORE_TEMP_FILES')
g_vars.syncFilePrg = os.path.join(g_vars.qa_src, 'bin', 'syncFiles.x')
g_vars.checkPrg = os.path.join(g_vars.qa_src, 'bin',
'qA-' + qaOpts.getOpt('PROJECT_AS') + '.x')
if not os.access(g_vars.syncFilePrg, os.X_OK):
print g_vars.syncFilePrg + ' is not executable'
sys.exit(1)
if not os.access(g_vars.checkPrg, os.X_OK):
print g_vars.checkPrg + ' is not executable'
sys.exit(1)
g_vars.anyProgress = False
return
def prepareExample(qaConf):
if not qaConf.isOpt('PROJECT'):
qaConf.addOpt("PROJECT", 'CORDEX')
if qaConf.isOpt('WORK'):
currdir = qaConf.getOpt('WORK')
else:
currdir = qaConf.getOpt('CURR_DIR')
#currdir=os.path.join(currdir, 'example')
qaConf.dOpts['QA_RESULTS'] = os.path.join(currdir, 'results')
if not qa_util.mkdirP(currdir):
sys.exit(1)
os.chdir(currdir)
qa_util.rm_r('results', 'config.txt', 'qa-test.task')
print 'make examples in ' + currdir
print 'make qa_test.task'
taskFile = os.path.join(QA_SRC, 'example', 'templates', 'qa-test.task')
shutil.copy(taskFile, currdir)
taskFile = 'qa-test.task'
# replace templates within QA_SRC/example
sub = []
repl = []
sub.append('PROJECT_DATA=data')
repl.append('PROJECT_DATA=' + os.path.join(currdir, 'data'))
sub.append('QA_RESULTS=results')
repl.append('QA_RESULTS=' + os.path.join(currdir, 'results'))
qa_util.f_str_replace(taskFile, sub, repl)
if not qa_util.which("ncgen"):
print "building data in example requires the ncgen utility"
sys.exit(1)
if not os.path.isdir(os.path.join(currdir, 'data')):
# data
print 'make data'
p = os.path.join(QA_SRC, 'example', 'templates', 'data.tbz')
subprocess.call(["tar", "--bzip2", "-xf", p])
for rs, ds, fs in os.walk('data'):
for f in fs:
nc_f = f[:len(f) - 3] + 'nc'
t_f = os.path.join(rs, f)
t_nc = os.path.join(rs, nc_f)
try:
subprocess.call(["ncgen", "-k", "3", "-o", t_nc, t_f])
except:
print 'making of example failed'
sys.exit(1)
else:
qa_util.rm_r(t_f)
print 'run' + sys.argv[0] + " -m -f " + currdir + "/qa-test.task"
qaConf = QaConfig(QA_SRC, ['-m', '-f', currdir + "/qa-test.task"])
return qaConf
def runExample():
if qaOpts.isOpt('EXAMPLE_PATH'):
currdir = os.path.join( qaOpts.getOpt('EXAMPLE_PATH'), 'example')
if currdir[0] != '/':
currdir = os.path.join(os.getcwd(), currdir)
else:
currdir=os.path.join(QA_SRC, 'example')
if not qa_util.mkdirP(currdir):
print 'could not mkdir ' + dir + ', please use option --example=path'
sys.exit(1)
os.chdir(currdir)
qa_util.rmR( 'results', 'config.txt', 'data', 'tables', 'qa-test.task' )
print 'make examples in ' + currdir
print 'make qa_test.task'
taskFile = os.path.join(QA_SRC, 'example', 'templates', 'qa-test.task')
shutil.copy( taskFile, currdir)
taskFile = 'qa-test.task'
# replace templates
sub=[]
repl=[]
sub.append('PROJECT_DATA=data')
repl.append('PROJECT_DATA=' + os.path.join(currdir, 'data') )
sub.append('QA_RESULTS=results')
repl.append('QA_RESULTS=' + os.path.join(currdir, 'results') )
qa_util.f_str_replace(taskFile, sub, repl)
# data
print 'make data'
subprocess.call(["tar", "--bzip2", "-xf", \
os.path.join(QA_SRC,os.path.join('example', 'templates', 'data.tbz') ) ])
txtFs=[]
for rs, ds, fs in os.walk('data'):
for f in fs:
if '.txt' in f:
txtFs.append(os.path.join(rs,f))
if qa_util.which("ncgen"):
for f in txtFs:
nc_f = f[:len(f)-2] + 'nc'
subprocess.call(["ncgen", "-k", "3", "-o", nc_f, f])
qa_util.rmR(f)
else:
print "building data in example requires the ncgen utility"
print 'run'
print os.path.join(QA_SRC, 'scripts', 'qa-dkrz') +\
" -m --work=" + currdir + '-f qa-test.task'
subprocess.call([os.path.join(QA_SRC, 'scripts', 'qa-dkrz'), \
'--work=' + currdir, "-f", "qa-test.task"])
return
