def get_all_logfiles():
f_logs=[]
while True:
try:
# fBase: list of filename bases corresponding to variables;
# usually a single one. F
# fNames: corresponding sub-temporal files
data_path, fBase, fNames = getPaths.next()
except StopIteration:
break
else:
sub_path = data_path[g_vars.prj_dp_len+1:]
f_logs.append( qa_util.get_experiment_name(g_vars, qaOpts, fB=fBase,
sp=sub_path) )
return f_logs
def get_all_logfiles():
f_logs = []
if qaConf.isOpt('SHOW_EXP'):
isShowExp = True
lfn_var = {}
while True:
try:
# fBase: list of filename bases corresponding to variables;
# usually a single one. F
# fNames: corresponding sub-temporal files
data_path, fBase, fNames = getPaths.next()
except StopIteration:
break
else:
sub_path = data_path[g_vars.prj_dp_len + 1:]
f_log = qa_util.get_experiment_name(g_vars,
qaConf,
fB=fBase,
sp=sub_path)
if isShowExp:
if f_log in lfn_var:
if not sub_path in lfn_var[f_log]:
lfn_var[f_log].append(sub_path)
else:
lfn_var[f_log] = [sub_path]
f_logs.append(f_log)
if isShowExp:
for key in lfn_var:
print '\nLogfile-name: ' + key
for p in lfn_var[key]:
print ' Variable: ' + p
sys.exit(0)
return f_logs
def run(log, g_vars, qaConf):
#g_vars.TTY = os.ttyname(0)
# update external tables and in case of running qa_dkrz.py from
# sources update C++ executables
# run_install(qaConf)
if qaConf.isOpt('NUM_EXEC_THREADS'):
g_vars.thread_num = \
sum( qa_util.mk_list(qaConf.getOpt('NUM_EXEC_THREADS')) )
else:
g_vars.thread_num = 1
g_vars.res_dir_path = qaConf.getOpt('QA_RESULTS')
g_vars.project_data_path = qaConf.getOpt('PROJECT_DATA')
g_vars.prj_dp_len = len(g_vars.project_data_path)
init_session(g_vars, qaConf)
g_vars.check_logs_path = os.path.join(g_vars.res_dir_path, 'check_logs')
g_vars.cs_enable = False
if qaConf.isOpt('CHECKSUM'):
g_vars.cs_enable = True
if qaConf.isOpt('CHECKSUM'):
g_vars.cs_type = 'md5'
else:
g_vars.cs_type = qaConf.getOpt('CHECKSUM')
cs_dir = qaConf.getOpt('CS_DIR')
if len(cs_dir) == 0:
cs_dir = 'cs_table'
g_vars.cs_dir = os.path.join(g_vars.res_dir_path, cs_dir)
qaConf.addOpt('LOG_FNAME_DIR', g_vars.check_logs_path)
qa_util.mkdirP(g_vars.check_logs_path) # error --> exit
qa_util.mkdirP(os.path.join(g_vars.res_dir_path, 'data')) # error --> exit
# some more settings
if not qaConf.isOpt('ZOMBIE_LIMIT'):
qaConf.addOpt('ZOMBIE_LIMIT', 3600)
if not qaConf.isOpt('CHECKSUM'):
if qaConf.isOpt('CS_STAND_ALONE') or qaConf.isOpt('CS_DIR'):
qaConf.addOpt('CHECKSUM', True)
# save current version id to the cfg-file
'''
if qaConf.isOpt('QA_VERSION'):
qv=qaConf.getOpt('QA_VERSION')
else:
qv = qa_util.get_curr_revision(g_vars.qa_src, g_vars.isConda)
qaConf.cfg.entry(key='QA_VERSION', value=qv)
g_vars.qa_revision = qv
'''
# abort criteria
isAbort = False
abortStr = ''
if len(qaConf.getOpt('PROJECT')) == 0:
isAbort = True
abortStr = 'PROJECT'
elif not qaConf.isOpt('PROJECT_DATA'):
if not qaConf.isOpt('SELECT_PATH_LIST'):
isAbort = True
abortStr = 'PROJECT_DATA or explicit filename'
if isAbort:
print abortStr + ' was not defined.'
print ' Did you miss to provide any of the sufficient options,'
print ' e.g. -f task-file, QA_CONF, -P PROJECT + filename?'
sys.exit(1)
# table path and copy of tables for operational runs
init_tables(g_vars, qaConf)
# unique exp_name and table_names are defined by indices of path components
g_vars.drs_path_base = qaConf.getOpt('DRS_PATH_BASE')
qa_util.get_experiment_name(g_vars, qaConf, isInit=True)
qa_util.get_project_table_name(g_vars, qaConf, isInit=True)
# enable clearance of logfile entries by the CLEAR option
if qaConf.isOpt('CLEAR_LOGFILE'):
g_vars.clear_logfile = True
else:
g_vars.clear_logfile = False
g_vars.ignore_temp_files = qaConf.isOpt('IGNORE_TEMP_FILES')
g_vars.syncFilePrg = os.path.join(g_vars.qa_src, 'bin', 'syncFiles.x')
g_vars.validNcPrg = os.path.join(g_vars.qa_src, 'bin', 'testValidNC.x')
g_vars.checkPrg = os.path.join(g_vars.qa_src, 'bin',
'qA-' + qaConf.getOpt('PROJECT') + '.x')
if not os.access(g_vars.syncFilePrg, os.X_OK):
print g_vars.syncFilePrg + ' is not executable'
sys.exit(1)
if not os.access(g_vars.validNcPrg, os.X_OK):
print g_vars.validNcPrg + ' is not executable'
sys.exit(1)
if not os.access(g_vars.checkPrg, os.X_OK):
print g_vars.checkPrg + ' is not executable'
sys.exit(1)
g_vars.anyProgress = False
return
def get_next_variable(data_path, fBase, fNames):
# test for parameters QUERY_..., if set at all
if query_files(qaConf, g_vars.validNcPrg, data_path, fNames):
return []
t_vars.sub_path = data_path[g_vars.prj_dp_len + 1:]
t_vars.var_path = os.path.join(g_vars.res_dir_path, 'data',
t_vars.sub_path)
# any QA result file of a previous check?
t_vars.qaFN = 'qa_' + fBase + '.nc'
qaNc = os.path.join(t_vars.var_path, t_vars.qaFN)
# experiment_name
t_vars.log_fname = \
qa_util.get_experiment_name(g_vars, qaConf, fB=fBase,
sp=t_vars.sub_path)
t_vars.pt_name = \
qa_util.get_project_table_name(g_vars, qaConf, fB=fBase,
sp=t_vars.sub_path)
# Any locks? Any clearance of previous results?
if testLock(t_vars, fBase):
return []
syncCall = g_vars.syncFilePrg + ' --only-marked'
if qaConf.isOpt('TEST_FNAME_ALIGNMENT'):
syncCall += ' --fname-alignment --continuous'
if os.path.exists(qaNc):
syncCall += ' -p ' + qaNc
else:
qaNc = ''
if qaConf.isOpt('TIME_LIMIT'):
syncCall += ' -l ' + qaConf.getOpt('TIME_LIMIT')
syncCall += ' -P ' + data_path
for f in fNames:
syncCall += ' ' + f
try:
next_file_str = subprocess.check_output(syncCall, shell=True)
except subprocess.CalledProcessError as e:
status = e.returncode
if status == 1:
return []
info = e.output.splitlines()
isLog = False
if status == 4:
# a fixed variable?
if len(qaNc):
# previously processed
return []
elif len(fNames) == 1:
next_file_str = fNames[0]
elif status == 3 or status == 5 or status == 6:
g_vars.anyProgress = True
if status == 3:
annot = 'Unspecific failure across sub-temporal file sequence'
tag = 'M5'
concl = 'DRS(F): FAIL'
elif status == 5:
annot = 'File(s) with invalid time data'
tag = 'M9'
concl = 'TIME: FAIL'
info = []
elif status == 6:
annot = 'Invalid NetCDF file'
tag = 'M0'
concl = 'CF: FAIL, CV: FAIL, DATA: FAIL, DRS: PASS, TIME: FAIL'
info = []
entry_id = log.append(t_vars.log_fname,
f=fBase,
d_path=data_path,
r_path=g_vars.res_dir_path,
sub_path=t_vars.sub_path,
annotation=annot,
impact='L2',
tag=tag,
info=info,
conclusion=concl,
status=status)
isLog = True
elif status > 8:
g_vars.anyProgress = True
entry_id = log.append(
t_vars.log_fname,
f=fBase,
d_path=data_path,
r_path=g_vars.res_dir_path,
sub_path=t_vars.sub_path,
annotation='ambiguous sub-temporal file sequence',
impact='L2',
tag='M5',
info=info,
conclusion='DRS(F): FAIL',
status=status)
isLog = True
if isLog:
log.write_entry(entry_id, g_vars.check_logs_path, t_vars.log_fname)
return []
try:
lst = next_file_str.splitlines()
except:
lst = []
if qaConf.isOpt('TEST_FNAME_ALIGNMENT'):
lst = []
return lst
def run(log, g_vars, qaOpts, rawCfgPars, cfgFile):
#g_vars.TTY = os.ttyname(0)
# are git and wget available?
g_vars.NO_GIT=True
if qa_util.which('git'):
g_vars.NO_GIT = False
g_vars.NO_WGET=True
if qa_util.which('wget'):
g_vars.NO_WGET = False
# update external tables and in case of running qa_dkrz.py from
# sources update C++ executables
run_install(qaOpts, g_vars)
if qaOpts.isOpt('NUM_EXEC_THREADS'):
g_vars.thread_num = \
sum( qa_util.mk_list(qaOpts.getOpt('NUM_EXEC_THREADS')) )
else:
g_vars.thread_num = 1
g_vars.res_dir_path = qaOpts.getOpt('QA_RESULTS')
g_vars.project_data_path = qaOpts.getOpt('PROJECT_DATA')
g_vars.prj_dp_len = len(g_vars.project_data_path)
init_session(g_vars, qaOpts)
g_vars.check_logs_path = os.path.join(g_vars.res_dir_path, 'check_logs')
g_vars.cs_enable = False
if qaOpts.isOpt('CHECKSUM'):
g_vars.cs_enable = True
if qaOpts.isOpt('CHECKSUM', True):
g_vars.cs_type = 'md5'
else:
g_vars.cs_type = qaOpts.getOpt('CHECKSUM')
cs_dir = qaOpts.getOpt('CS_DIR')
if len(cs_dir) == 0:
cs_dir='cs_table'
g_vars.cs_dir = os.path.join(g_vars.res_dir_path, cs_dir)
qaOpts.setOpt('LOG_FNAME_DIR', g_vars.check_logs_path)
qa_util.mkdirP(g_vars.check_logs_path) # error --> exit
qa_util.mkdirP(os.path.join(g_vars.res_dir_path, 'data')) # error --> exit
# some more settings
if not qaOpts.isOpt('ZOMBIE_LIMIT'):
qaOpts.setOpt('ZOMBIE_LIMIT', 3600)
if not qaOpts.isOpt('CHECKSUM'):
if qaOpts.isOpt('CS_STAND_ALONE') or qaOpts.isOpt('CS_DIR'):
qaOpts.setOpt('CHECKSUM', True)
# save current version id to the cfg-file
qv=qaOpts.getOpt('QA_REVISION')
if len(qv) == 0:
qv=qa_util.get_curr_revision(g_vars.qa_src, g_vars.isConda)
qv = qa_util.cfg_parser(rawCfgPars, cfgFile,
section=g_vars.qa_src, key='QA_REVISION', value=qv)
qaOpts.setOpt('QA_REVISION', qv)
g_vars.qa_revision = qv
# table path and copy of tables for operational runs
init_tables(g_vars, qaOpts)
# unique exp_name and table_names are defined by indices of path components
qa_util.get_experiment_name(g_vars, qaOpts, isInit=True)
qa_util.get_project_table_name(g_vars, qaOpts, isInit=True)
# enable clearance of logfile entries by the CLEAR option
if qaOpts.isOpt('CLEAR_LOGFILE'):
g_vars.clear_logfile = True
else:
g_vars.clear_logfile = False
g_vars.ignore_temp_files = qaOpts.isOpt('IGNORE_TEMP_FILES')
g_vars.syncFilePrg = os.path.join(g_vars.qa_src, 'bin', 'syncFiles.x')
g_vars.checkPrg = os.path.join(g_vars.qa_src, 'bin',
'qA-' + qaOpts.getOpt('PROJECT_AS') + '.x')
if not os.access(g_vars.syncFilePrg, os.X_OK):
print g_vars.syncFilePrg + ' is not executable'
sys.exit(1)
if not os.access(g_vars.checkPrg, os.X_OK):
print g_vars.checkPrg + ' is not executable'
sys.exit(1)
g_vars.anyProgress = False
return
def get_next_variable(data_path, fBase, fNames):
t_vars.sub_path = data_path[g_vars.prj_dp_len+1:]
t_vars.var_path = os.path.join(g_vars.res_dir_path, 'data',
t_vars.sub_path )
# any QA result file of a previous check?
t_vars.qaFN = 'qa_' + fBase + '.nc'
qaNc = os.path.join(t_vars.var_path, t_vars.qaFN)
# experiment_name
t_vars.log_fname = \
qa_util.get_experiment_name(g_vars, qaOpts, fB=fBase,
sp=t_vars.sub_path)
t_vars.pt_name = \
qa_util.get_project_table_name(g_vars, qaOpts, fB=fBase,
sp=t_vars.sub_path)
# Any locks? Any clearance of previous results?
if testLock(t_vars, fBase):
return []
syncCall = g_vars.syncFilePrg + ' --only-marked'
if os.path.exists(qaNc):
syncCall += ' -p ' + qaNc
if qaOpts.isOpt('TIME_LIMIT'):
syncCall += ' -l ' + qaOpts.getOpts('TIME_LIMIT')
syncCall += ' -P ' + data_path
for f in fNames:
syncCall += ' ' + f
try:
next_file_str = subprocess.check_output(syncCall, shell=True)
except subprocess.CalledProcessError as e:
status = e.returncode
if status == 1:
return []
info = e.output.splitlines()
isLog=False
if status == 3:
g_vars.anyProgress = True
entry_id = log.append(t_vars.log_fname, f=fBase, d_path=data_path,
r_path=g_vars.res_dir_path, sub_path=t_vars.sub_path,
caption='unspecific error in sub-temporal file sequence',
impact='L2', tag='M5', info=info,
conclusion='DRS(F): FAIL', status=status)
isLog = True
elif status == 4:
# a fixed variable?
if len(qaNc):
# previously processed
return []
elif status > 10:
g_vars.anyProgress = True
entry_id = log.append( t_vars.log_fname, f=fBase, d_path=data_path,
r_path=g_vars.res_dir_path, sub_path=t_vars.sub_path,
caption='ambiguous sub-temporal file sequence',
impact='L2', tag='M6', info=info,
conclusion='DRS(F): FAIL', status=status)
isLog = True
if isLog:
log.write_entry(entry_id,
g_vars.check_logs_path,
t_vars.log_fname)
return []
lst = next_file_str.splitlines()
return lst