Exemplo n.º 1
0
    def __init__(self, log, qaConf, g_vars):
        # connection of any executing entity, here for QA-DKRZ
        self.qa_exec = QaExec(log, qaConf, g_vars)

        self.g_vars = g_vars

        self.is_next=False
        if qaConf.isOpt('NEXT'):
            self.is_next=True
            self.next=qaConf.getOpt('NEXT')
Exemplo n.º 2
0
    def __init__(self, log, qaOpts, g_vars):
        # connection of any executing entity, here for QA-DKRZ
        self.qa_exec = QaExec(log, qaOpts, g_vars)

        self.g_vars = g_vars

        self.is_next=False
        if qaOpts.isOpt('NEXT'):
            self.is_next=True
            self.next=qaOpts.getOpt('NEXT')
Exemplo n.º 3
0
def run():
    if qaConf.isOpt("SHOW_EXP"):
        f_log = get_all_logfiles()
        sys.exit(0)

    if qaConf.isOpt('SHOW') or qaConf.isOpt('NEXT'):
        g_vars.thread_num = 1

    # the queue is two items longer than the number of threads
    queue = Queue(maxsize=g_vars.thread_num + 2)

    launch_list = []

    if g_vars.thread_num < 2:
        # a single thread
        qaExec = QaExec(log, qaConf, g_vars)
        launch_list.append(qaExec)
    else:
        for i in range(g_vars.thread_num):
            launch_list.append(QaLauncher(log, qaConf, g_vars))

        for i in range(g_vars.thread_num):
            t = Thread(target=launch_list[i].start, args=(queue, ))
            t.daemon = True
            t.start()

    is_next_var = False
    if qaConf.isOpt('NEXT_VAR'):
        is_next_var = True
        next_var = qaConf.getOpt('NEXT_VAR')
        count_next_var = 0

    while True:

        if qaConf.isOpt('EXPLICIT_FILES'):
            if len(qaConf.dOpts['EXPLICIT_FILES']):
                data_path, f = os.path.split(qaConf.dOpts['EXPLICIT_FILES'][0])
                t_r = qa_util.f_time_range(f)
                fBase = t_r[0]
                fNames = [f]
                del qaConf.dOpts['EXPLICIT_FILES'][0]
            else:
                queue.put(('---EOQ---', '', t_vars), block=True)
                break
        else:
            if is_next_var:
                if count_next_var == next_var:
                    break

                count_next_var += 1

            try:
                # fBase: list of filename bases corresponding to variables;
                #        usually a single one. F
                # fNames: corresponding sub-temporal files
                data_path, fBase, fNames = getPaths.next()

            except StopIteration:
                queue.put(('---EOQ---', '', t_vars), block=True)
                break
            else:
                isNoPath = False

        # return a list of files which have not been processed, yet.
        # Thus, the list could be empty
        fL = get_next_variable(data_path, fBase, fNames)

        if len(fL) == 0:
            continue

        t_vars.fBase = fBase

        #queue.put( (data_path, fL, t_vars), block=True)
        queue.put((data_path, fL, copy.deepcopy(t_vars)), block=True)

        if g_vars.thread_num < 2:
            # a single thread
            if not launch_list[0].start(queue, ):
                break

    if g_vars.thread_num > 1:
        queue.join()

    if g_vars.thread_num < 2:
        launch_list[0].printStatusLine()
    else:
        launch_list[0].qa_exec.printStatusLine()

    return
Exemplo n.º 4
0
class QaLauncher(object):
    '''
    classdocs
    '''
    task_finished = False

    def __init__(self, log, qaOpts, g_vars):
        # connection of any executing entity, here for QA-DKRZ
        self.qa_exec = QaExec(log, qaOpts, g_vars)

        self.g_vars = g_vars

        self.is_next=False
        if qaOpts.isOpt('NEXT'):
            self.is_next=True
            self.next=qaOpts.getOpt('NEXT')


    def run(self, data_path, fName, t_vars):
        # print fName
        return self.qa_exec.run(t_vars)


    def start(self, queue):

        while True:
            if QaLauncher.task_finished:
                break

            try:
                (data_path, fNames, t_vars) = queue.get()
            except:
                pass
            else:
                if data_path == '---EOQ---':
                    queue.task_done()
                    QaLauncher.task_finished=True
                    break


                lenFN_1=len(fNames)-1

                for ix in range(lenFN_1+1):
                    is_post_proc = False

                    if self.is_next:
                        self.next -= 1
                        if self.next == -1:
                            break

                    if ix == 0:
                        if ix == lenFN_1:
                            seq_pos = 'x'  # there is only a single file
                            is_post_proc = True
                        else:
                            seq_pos = 'f'
                    elif ix == lenFN_1:
                        seq_pos = 'l'
                        is_post_proc = True
                    else:
                        seq_pos = 's'

                    t_vars.data_path = data_path
                    t_vars.fName = fNames[ix]
                    t_vars.seq_pos = seq_pos
                    t_vars.post_proc = is_post_proc

                    if not self.run(data_path, fNames[ix], t_vars):
                        break

                # for the previous range == 1
                if self.is_next and self.next == 0:
                    return False

            queue.task_done()


        return
Exemplo n.º 5
0
class QaLauncher(object):
    '''
    classdocs
    '''
    task_finished = False

    def __init__(self, log, qaConf, g_vars):
        # connection of any executing entity, here for QA-DKRZ
        self.qa_exec = QaExec(log, qaConf, g_vars)

        self.g_vars = g_vars

        self.is_next=False
        if qaConf.isOpt('NEXT'):
            self.is_next=True
            self.next=qaConf.getOpt('NEXT')


    def run(self, data_path, fName, t_vars):
        # print fName
        return self.qa_exec.run(t_vars)


    def start(self, queue):

        while True:
            if QaLauncher.task_finished:
                break

            try:
                (data_path, fNames, t_vars) = queue.get()
            except:
                pass
            else:
                if data_path == '---EOQ---':
                    queue.task_done()
                    QaLauncher.task_finished=True
                    break


                lenFN_1=len(fNames)-1

                for ix in range(lenFN_1+1):
                    is_post_proc = False

                    if self.is_next:
                        self.next -= 1
                        if self.next == -1:
                            break

                    if ix == 0:
                        if ix == lenFN_1:
                            seq_pos = 'x'  # there is only a single file
                            is_post_proc = True
                        else:
                            seq_pos = 'f'
                    elif ix == lenFN_1:
                        seq_pos = 'l'
                        is_post_proc = True
                    else:
                        seq_pos = 's'

                    t_vars.data_path = data_path
                    t_vars.fName = fNames[ix]
                    t_vars.seq_pos = seq_pos
                    t_vars.post_proc = is_post_proc

                    if not self.run(data_path, fNames[ix], t_vars):
                        break

                # for the previous range == 1
                if self.is_next and self.next == 0:
                    return False

            queue.task_done()


        return