def __init__(self, log, qaConf, g_vars):
# connection of any executing entity, here for QA-DKRZ
self.qa_exec = QaExec(log, qaConf, g_vars)
self.g_vars = g_vars
self.is_next=False
if qaConf.isOpt('NEXT'):
self.is_next=True
self.next=qaConf.getOpt('NEXT')
def __init__(self, log, qaOpts, g_vars):
# connection of any executing entity, here for QA-DKRZ
self.qa_exec = QaExec(log, qaOpts, g_vars)
self.g_vars = g_vars
self.is_next=False
if qaOpts.isOpt('NEXT'):
self.is_next=True
self.next=qaOpts.getOpt('NEXT')
def run():
if qaConf.isOpt("SHOW_EXP"):
f_log = get_all_logfiles()
sys.exit(0)
if qaConf.isOpt('SHOW') or qaConf.isOpt('NEXT'):
g_vars.thread_num = 1
# the queue is two items longer than the number of threads
queue = Queue(maxsize=g_vars.thread_num + 2)
launch_list = []
if g_vars.thread_num < 2:
# a single thread
qaExec = QaExec(log, qaConf, g_vars)
launch_list.append(qaExec)
else:
for i in range(g_vars.thread_num):
launch_list.append(QaLauncher(log, qaConf, g_vars))
for i in range(g_vars.thread_num):
t = Thread(target=launch_list[i].start, args=(queue, ))
t.daemon = True
t.start()
is_next_var = False
if qaConf.isOpt('NEXT_VAR'):
is_next_var = True
next_var = qaConf.getOpt('NEXT_VAR')
count_next_var = 0
while True:
if qaConf.isOpt('EXPLICIT_FILES'):
if len(qaConf.dOpts['EXPLICIT_FILES']):
data_path, f = os.path.split(qaConf.dOpts['EXPLICIT_FILES'][0])
t_r = qa_util.f_time_range(f)
fBase = t_r[0]
fNames = [f]
del qaConf.dOpts['EXPLICIT_FILES'][0]
else:
queue.put(('---EOQ---', '', t_vars), block=True)
break
else:
if is_next_var:
if count_next_var == next_var:
break
count_next_var += 1
try:
# fBase: list of filename bases corresponding to variables;
# usually a single one. F
# fNames: corresponding sub-temporal files
data_path, fBase, fNames = getPaths.next()
except StopIteration:
queue.put(('---EOQ---', '', t_vars), block=True)
break
else:
isNoPath = False
# return a list of files which have not been processed, yet.
# Thus, the list could be empty
fL = get_next_variable(data_path, fBase, fNames)
if len(fL) == 0:
continue
t_vars.fBase = fBase
#queue.put( (data_path, fL, t_vars), block=True)
queue.put((data_path, fL, copy.deepcopy(t_vars)), block=True)
if g_vars.thread_num < 2:
# a single thread
if not launch_list[0].start(queue, ):
break
if g_vars.thread_num > 1:
queue.join()
if g_vars.thread_num < 2:
launch_list[0].printStatusLine()
else:
launch_list[0].qa_exec.printStatusLine()
return
class QaLauncher(object):
'''
classdocs
'''
task_finished = False
def __init__(self, log, qaOpts, g_vars):
# connection of any executing entity, here for QA-DKRZ
self.qa_exec = QaExec(log, qaOpts, g_vars)
self.g_vars = g_vars
self.is_next=False
if qaOpts.isOpt('NEXT'):
self.is_next=True
self.next=qaOpts.getOpt('NEXT')
def run(self, data_path, fName, t_vars):
# print fName
return self.qa_exec.run(t_vars)
def start(self, queue):
while True:
if QaLauncher.task_finished:
break
try:
(data_path, fNames, t_vars) = queue.get()
except:
pass
else:
if data_path == '---EOQ---':
queue.task_done()
QaLauncher.task_finished=True
break
lenFN_1=len(fNames)-1
for ix in range(lenFN_1+1):
is_post_proc = False
if self.is_next:
self.next -= 1
if self.next == -1:
break
if ix == 0:
if ix == lenFN_1:
seq_pos = 'x' # there is only a single file
is_post_proc = True
else:
seq_pos = 'f'
elif ix == lenFN_1:
seq_pos = 'l'
is_post_proc = True
else:
seq_pos = 's'
t_vars.data_path = data_path
t_vars.fName = fNames[ix]
t_vars.seq_pos = seq_pos
t_vars.post_proc = is_post_proc
if not self.run(data_path, fNames[ix], t_vars):
break
# for the previous range == 1
if self.is_next and self.next == 0:
return False
queue.task_done()
return
class QaLauncher(object):
'''
classdocs
'''
task_finished = False
def __init__(self, log, qaConf, g_vars):
# connection of any executing entity, here for QA-DKRZ
self.qa_exec = QaExec(log, qaConf, g_vars)
self.g_vars = g_vars
self.is_next=False
if qaConf.isOpt('NEXT'):
self.is_next=True
self.next=qaConf.getOpt('NEXT')
def run(self, data_path, fName, t_vars):
# print fName
return self.qa_exec.run(t_vars)
def start(self, queue):
while True:
if QaLauncher.task_finished:
break
try:
(data_path, fNames, t_vars) = queue.get()
except:
pass
else:
if data_path == '---EOQ---':
queue.task_done()
QaLauncher.task_finished=True
break
lenFN_1=len(fNames)-1
for ix in range(lenFN_1+1):
is_post_proc = False
if self.is_next:
self.next -= 1
if self.next == -1:
break
if ix == 0:
if ix == lenFN_1:
seq_pos = 'x' # there is only a single file
is_post_proc = True
else:
seq_pos = 'f'
elif ix == lenFN_1:
seq_pos = 'l'
is_post_proc = True
else:
seq_pos = 's'
t_vars.data_path = data_path
t_vars.fName = fNames[ix]
t_vars.seq_pos = seq_pos
t_vars.post_proc = is_post_proc
if not self.run(data_path, fNames[ix], t_vars):
break
# for the previous range == 1
if self.is_next and self.next == 0:
return False
queue.task_done()
return
