Ejemplo n.º 1
0
    def prepare_pbs_files(self, pbs_module):
        """
        :type pbs_module: scripts.pbs.modules.pbs_tarkil_cesnet_cz
        :rtype: list[(str, PBSModule)]
        """

        jobs = list()

        for p in self.proc:
            case = ConfigCase(dict(
                proc=p,
                time_limit=self.time_limit,
                memory_limit=self.memory_limit,
                tmp='exec-parallel'
            ), None)

            pbs_run = pbs_module.Module(case)
            pbs_run.queue = self.arg_options.get('queue', True)
            pbs_run.ppn = self.arg_options.get('ppn', 1)

            pbs_content = self.create_pbs_job_content(pbs_module, case)
            IO.write(case.fs.pbs_script, pbs_content)

            qsub_command = pbs_run.get_pbs_command(case.fs.pbs_script)
            jobs.append((qsub_command, pbs_run))
        return jobs
Ejemplo n.º 2
0
def finish_pbs_exec(job, batch):
    """
    Upon PBS finish determine Job exit
    :rtype: scripts.serialization.PyPyResult
    :type job: scripts.pbs.job.Job
    """
    job.is_active = False
    try:
        result = load_pypy(job.case.fs.dump_output)
    except Exception as e:
        # no output file was generated assuming it went wrong
        job.status = JobState.EXIT_ERROR
        Printer.all.err('Job {} ended (no output file found). Case: {}', job, job.full_name)
        Printer.all.out('       pbs output: ')
        Printer.all.raw(format_n_lines(IO.read(job.case.fs.pbs_output), False))
        return

    # check result
    if result.returncode == 0:
        job.status = JobState.EXIT_OK
        Printer.all.suc('Job {}({}) ended', job, job.full_name)
    else:
        job.status = JobState.EXIT_ERROR
        Printer.all.err('Job {}({}) ended', job, job.full_name)

    if result.returncode != 0 or batch:
        with Printer.all.with_level(1):
            Printer.all.raw(format_n_lines(IO.read(result.output), result.returncode == 0))

    return result
Ejemplo n.º 3
0
def finish_pbs_runtest(job, batch):
    """
    Upon PBS runtest finish determine Job exit
    :type job: scripts.pbs.job.Job
    :rtype: ResultParallelThreads
    """
    job.is_active = False
    try:
        runner = load_runtest(job.case.fs.dump_output)
    except:
        # no output file was generated assuming it went wrong
        job.status = JobState.EXIT_ERROR
        Printer.all.err('Job {} ended (no output file found). Case: {}', job, job.full_name)
        Printer.all.out('       pbs output: ')
        Printer.all.raw(format_n_lines(IO.read(job.case.fs.pbs_output), False))
        return

    job.status = JobState.EXIT_OK if runner.returncode == 0 else JobState.EXIT_ERROR

    for thread in runner.threads:
        StatusPrinter.print_test_result(thread)
        if thread.returncode != 0 or batch:
            with Printer.all.with_level():
                Printer.all.out('Log file {}', job.case.fs.job_output)
                Printer.all.raw(format_n_lines(IO.read(job.case.fs.job_output), thread.returncode == 0))

    # print status line only if pbs job container more cases
    if len(runner.threads) > 1:
        Printer.all.sep()
        StatusPrinter.print_runner_stat(runner)
        Printer.all.sep()

    return runner
Ejemplo n.º 4
0
    def compare(self, reference_filepath, other_filepath, **kwargs):
        """
        Method can do anything as long as int value is returned
        :param reference_filepath:
        :param other_filepath:
        :param kwargs:
        :return:
        """

        reference_content = IO.read(
            Paths.abspath(reference_filepath)
        )
        other_content = IO.read(
            Paths.abspath(other_filepath)
        )

        self.output.write("In case of emergency,")
        self.output.write("    you can provide details on what went wrong")
        self.output.write("    using self.output.write method")
        self.output.write("")
        self.output.write("Error while comparing files \n{} \n{}"
                          .format(reference_filepath, other_filepath))

        # must return return-code!
        return 1
Ejemplo n.º 5
0
    def on_complete(self, pypy=None):
        if pypy.executor.broken:
            return

        if self.log_file:
            IO.write(self.log_file, self.content)

        enabled = False
        if self.policy == self.POLICY_ALWAYS:
            enabled = True
        elif self.policy == self.POLICY_BATCH_OR_ERROR:
            enabled = pypy.with_error() or (not Printer.batched.is_muted())
        elif self.policy == self.POLICY_ERROR_ONLY:
            enabled = pypy.with_error()

        if enabled:
            with Printer.all.with_level():

                if self.pypy.full_output:
                    Printer.all.sep()
                    Printer.all.out('Output from file {self.pypy.full_output}'.format(**locals()))

                if not Printer.batched.is_muted():
                    Printer.batched.raw(format_n_lines(self.content, pypy.was_successful()))

                if not Printer.console.is_muted():
                    Printer.console.raw(format_n_lines(self.content, pypy.was_successful()))
Ejemplo n.º 6
0
    def on_complete(self, pypy=None):
        if pypy.executor.broken:
            return

        if self.log_file:
            IO.write(self.log_file, self.content)

        enabled = False
        if self.policy == self.POLICY_ALWAYS:
            enabled = True
        elif self.policy == self.POLICY_BATCH_OR_ERROR:
            enabled = pypy.with_error() or (not Printer.batched.is_muted())
        elif self.policy == self.POLICY_ERROR_ONLY:
            enabled = pypy.with_error()

        if enabled:
            with Printer.all.with_level():

                if self.pypy.full_output:
                    Printer.all.sep()
                    Printer.all.out(
                        'Output from file {self.pypy.full_output}'.format(
                            **locals()))

                if not Printer.batched.is_muted():
                    Printer.batched.raw(
                        format_n_lines(self.content, pypy.was_successful()))

                if not Printer.console.is_muted():
                    Printer.console.raw(
                        format_n_lines(self.content, pypy.was_successful()))
    def test_pbs_mode(self):
        # this test can be only tested on PBS server

        do_work(parser, ['-q', '--', 'sleep', '1'], debug=True)
        mpi_dirs = [f for f in os.listdir(__dir__) if f.startswith('exec-parallel-')]
        self.assertGreaterEqual(len(mpi_dirs), 1)

        # test that script were actually created and contain at least one file
        # also remove them ...
        for mpi in mpi_dirs:
            self.assertGreaterEqual(len(os.listdir(os.path.join(__dir__, mpi))), 1)
            IO.delete_all(os.path.join(__dir__, mpi))
Ejemplo n.º 8
0
    def tearDownClass(cls):
        super(UnitTest, cls).tearDownClass()

        from scripts.core.base import IO
        cls.end_state = os.listdir(cls.__dir__)

        for f in cls.end_state:
            if f not in cls.start_state:
                full_path = os.path.join(cls.__dir__, f)
                if os.path.isdir(full_path):
                    IO.delete_all(full_path)
                else:
                    IO.delete(full_path)
Ejemplo n.º 9
0
    def tearDownClass(cls):
        super(UnitTest, cls).tearDownClass()

        from scripts.core.base import IO
        cls.end_state = os.listdir(cls.__dir__)

        for f in cls.end_state:
            if f not in cls.start_state:
                full_path = os.path.join(cls.__dir__, f)
                if os.path.isdir(full_path):
                    IO.delete_all(full_path)
                else:
                    IO.delete(full_path)
Ejemplo n.º 10
0
    def on_complete(self, pypy=None):

        if self.log_file:
            IO.write(self.log_file, self.content)

        # finish after update
        if self.update_format and (not self.complete_format
                                   and not self.color_complete_format):
            printf.finish_rewrite()

        # print regular messages
        for fmt in ensure_iterable(self.complete_format):
            printf.out(fmt, monitor=self)

        # print messages if error
        if self.pypy.with_error():
            for fmt in ensure_iterable(self.error_complete_format):
                printf.error(fmt, monitor=self)

        # print regular color messages based on result
        for fmt in ensure_iterable(self.color_complete_format):
            if self.pypy.returncode() == 0:
                printf.success(fmt, monitor=self)
            elif self.pypy.returncode() is None:
                printf.warning(fmt, monitor=self)
            else:
                printf.error(fmt, monitor=self)

        with printf:
            if printf.verbosity() is printf.OutputVerbosity.FULL:
                printf.sep()
                printf.out('Output from file {self.pypy.full_output}'.format(
                    **locals()))
                printf.opt(raw=True).stream(
                    format_n_lines(self.content, pypy.was_successful()))
            elif printf.verbosity(
            ) is printf.OutputVerbosity.SMART and pypy.with_error():
                printf.sep()
                printf.out(
                    'Last 50 lines from file {self.pypy.full_output}'.format(
                        **locals()))
                printf.opt(raw=True).stream(
                    format_n_lines(self.content, success=False, n_lines=-50))
            elif printf.verbosity(
            ) is printf.OutputVerbosity.MINIMAL and pypy.with_error():
                printf.sep()
                printf.out(
                    'Last 50 lines from file {self.pypy.full_output}'.format(
                        **locals()))
                printf.opt(raw=True).stream(
                    format_n_lines(self.content, success=False, n_lines=-50))
Ejemplo n.º 11
0
    def on_thread_complete(self, thread):
        """
        :type thread: scripts.core.pypy.PyPy
        """
        super(ComparisonMultiThread, self).on_thread_complete(thread)

        # append ndiff to file
        content = list()
        content.append('-' * 60 + '\n')
        content.append(thread.name + '\n')
        content.append('-' * 60 + '\n')
        content.append(thread.executor.output.read())
        content.append('\n' * 3)
        IO.append(self.output, '\n'.join(content) or '')
Ejemplo n.º 12
0
 def generate_status_file(cls, target):
     """
     Will generate status file if target has option turned on
     :type target: PyPy
     """
     if target.status_file:
         IO.write(target.status_file, json.dumps(target.status(), indent=4))
         output_dir = Paths.dirname(target.status_file)
         files = Paths.browse(
             output_dir,
             [PathFilters.filter_wildcards('*/profiler_info_*.log.json')])
         # profiler json is missing?
         if not files:
             IO.write(
                 Paths.join(output_dir, 'profiler_info_dummy.log.json'),
                 '{}')
Ejemplo n.º 13
0
    def prepare_pbs_files(self, pbs_module):
        jobs = list()
        """ :type: list[(str, PBSModule)] """

        for yaml_file, yaml_config in self.configs.files.items():
            for case in yaml_config.get_one(yaml_file):
                pbs_run = pbs_module.Module(case)
                pbs_run.queue = self.arg_options.get('queue', True)
                pbs_run.ppn = self.arg_options.get('ppn', 1)

                pbs_content = self.create_pbs_job_content(pbs_module, case)
                IO.write(case.fs.pbs_script, pbs_content)

                qsub_command = pbs_run.get_pbs_command(case.fs.pbs_script)
                jobs.append((qsub_command, pbs_run))
        return jobs
Ejemplo n.º 14
0
def finish_pbs_exec(job, batch):
    """
    Upon PBS finish determine Job exit
    :rtype: scripts.serialization.PyPyResult
    :type job: scripts.pbs.job.Job
    """
    job.is_active = False
    try:
        result = load_pypy(job.case.fs.dump_output)
    except Exception as e:
        # no output file was generated assuming it went wrong
        job.status = JobState.EXIT_ERROR
        Printer.all.err('Job {} ended (no output file found). Case: {}', job,
                        job.full_name)
        Printer.all.out('       pbs output: ')
        Printer.all.raw(format_n_lines(IO.read(job.case.fs.pbs_output), False))
        return

    # check result
    if result.returncode == 0:
        job.status = JobState.EXIT_OK
        Printer.all.suc('Job {}({}) ended', job, job.full_name)
    else:
        job.status = JobState.EXIT_ERROR
        Printer.all.err('Job {}({}) ended', job, job.full_name)

    if result.returncode != 0 or batch:
        with Printer.all.with_level(1):
            Printer.all.raw(
                format_n_lines(IO.read(result.output), result.returncode == 0))

    return result
Ejemplo n.º 15
0
    def read(self):
        if self.mode is self.DUMMY:
            return self.content

        if self.filename:
            if self.content is None:
                self.content = IO.read(self.filename)
            return self.content
Ejemplo n.º 16
0
    def print_log_file(cls, f, n_lines):
        log_file = IO.read(f)
        if log_file:
            if n_lines == 0:
                Printer.out('Full log from file {}:', f)
            else:
                Printer.out('Last {} lines from file {}:', abs(n_lines), f)

            Printer.wrn(format_n_lines(log_file.rstrip(), -n_lines, indent=Printer.indent * '    '))
Ejemplo n.º 17
0
    def print_log_file(cls, f, n_lines):
        log_file = IO.read(f)
        if log_file:
            if n_lines == 0:
                printf.out('Full log from file {}:', f)
            else:
                printf.out('Last {} lines from file {}:', abs(n_lines), f)

            printf.stream(format_n_lines(log_file.rstrip(), -n_lines))
Ejemplo n.º 18
0
    def print_log_file(cls, f, n_lines):
        log_file = IO.read(f)
        if log_file:
            if n_lines == 0:
                Printer.out('Full log from file {}:', f)
            else:
                Printer.out('Last {} lines from file {}:', abs(n_lines), f)

            Printer.wrn(
                format_n_lines(log_file.rstrip(),
                               -n_lines,
                               indent=Printer.indent * '    '))
Ejemplo n.º 19
0
    def close(self):
        if self.mode in {self.WRITE, self.APPEND, self.VARIABLE}:
            if self.fp is not None:
                if type(self.fp) is int:
                    os.close(self.fp)
                else:
                    self.fp.close()
                self.fp = None

        # remove temp file
        if self.mode in {self.VARIABLE}:
            self.content = IO.read(self.filename)
            os.unlink(self.filename)
def run_local_mode_one(proc, time_limit, memory_limit):
    if proc == 0:
        command = arg_rest[1:]
    else:
        command = [arg_rest[0], '-np', proc] + arg_rest[1:]

    n_lines = 0 if arg_options.batch else 10
    pypy = PyPy(BinExecutor(command))

    # set limits
    pypy.limit_monitor.time_limit = time_limit
    pypy.limit_monitor.memory_limit = memory_limit
    pypy.progress = not arg_options.batch
    pypy.info_monitor.deactivate()
    pypy.error_monitor.deactivate()

    # catch output to variable
    # in batch mode we will keep the files
    # otherwise we will keep logs only on error
    log_file = Paths.temp_file('exec-parallel-{date}-{time}-{rnd}.log')
    pypy.executor.output = OutputMode.variable_output()
    pypy.full_output = log_file

    # start and wait for exit
    pypy.start()
    pypy.join()

    # add result to global json result
    GlobalResult.add(pypy)

    # in batch mode or on error
    if not pypy.with_success() or arg_options.batch:
        content = pypy.executor.output.read()
        IO.write(log_file, content)
        Printer.close()
        Printer.out(format_n_lines(content, indent='    ', n_lines=-n_lines))
        Printer.open()
    return pypy
Ejemplo n.º 21
0
def finish_pbs_job(job, batch):
    """
    :type job: scripts.pbs.job.Job
    """
    # try to get more detailed job status
    job.is_active = False
    job_output = IO.read(job.case.fs.json_output)

    if job_output:
        job_json = JsonParser(json.loads(job_output), batch)
        if job_json.returncode == 0:
            job.status = JobState.EXIT_OK
            Printer.out('OK:    Job {}({}) ended', job, job.full_name)
            Printer.open()
            # in batch mode print all logs
            if batch:
                Printer.open()
                for test in job_json.tests:
                    test.get_result()
                Printer.close()
            Printer.close()
        else:
            job.status = JobState.EXIT_ERROR
            Printer.out('ERROR: Job {}({}) ended', job, job.full_name)
            # in batch mode print all logs

            Printer.open()
            for test in job_json.tests:
                test.get_result()
            Printer.close()
    else:
        # no output file was generated assuming it went wrong
        job.status = JobState.EXIT_ERROR
        Printer.out('ERROR: Job {} ended (no output file found). Case: {}', job, job.full_name)
        Printer.out('       pbs output: ')
        Printer.out(format_n_lines(IO.read(job.case.fs.pbs_output), 0))
    return 0 if job.status == JobState.EXIT_OK else 1
Ejemplo n.º 22
0
    def close(self):
        if self.mode is self.DUMMY:
            return

        if self.mode in {self.WRITE, self.APPEND, self.VARIABLE}:
            if self.fp is not None:
                if type(self.fp) is int:
                    os.close(self.fp)
                else:
                    self.fp.close()
                self.fp = None

        # remove temp file
        if self.mode in {self.VARIABLE}:
            self.content = IO.read(self.filename)
            os.unlink(self.filename)
Ejemplo n.º 23
0
 def read(self):
     if self.filename:
         if self.content is None:
             self.content = IO.read(self.filename)
         return self.content
def run_pbs_mode(configs, debug=False):
    """
    :type debug: bool
    :type configs: scripts.config.yaml_config.ConfigPool
    """
    global arg_options, arg_others, arg_rest
    pbs_module = get_pbs_module(arg_options.host)
    Printer.dynamic_output = not arg_options.batch
    Printer.dyn('Parsing yaml files')

    jobs = list()
    """ :type: list[(str, PBSModule)] """

    for yaml_file, yaml_config in configs.files.items():
        for case in yaml_config.get_one(yaml_file):
            pbs_run = pbs_module.Module(case)
            pbs_run.queue = arg_options.get('queue', True)
            pbs_run.ppn = arg_options.get('ppn', 1)

            pbs_content = create_pbs_job_content(pbs_module, case)
            IO.write(case.fs.pbs_script, pbs_content)

            qsub_command = pbs_run.get_pbs_command(case.fs.pbs_script)
            jobs.append((qsub_command, pbs_run))

    # start jobs
    Printer.dyn('Starting jobs')

    total = len(jobs)
    job_id = 0
    multijob = MultiJob(pbs_module.ModuleJob)
    for qsub_command, pbs_run in jobs:
        job_id += 1

        Printer.dyn('Starting jobs {:02d} of {:02d}', job_id, total)

        output = subprocess.check_output(qsub_command)
        job = pbs_module.ModuleJob.create(output, pbs_run.case)
        job.full_name = "Case {}".format(pbs_run.case)
        multijob.add(job)

    Printer.out()
    Printer.out('{} job/s inserted into queue', total)

    # # first update to get more info about multijob jobs
    Printer.out()
    Printer.separator()
    Printer.dyn('Updating job status')
    multijob.update()

    # print jobs statuses
    Printer.out()
    if not arg_options.batch:
        multijob.print_status()

    Printer.separator()
    Printer.dyn(multijob.get_status_line())
    returncodes = dict()

    # wait for finish
    while multijob.is_running():
        Printer.dyn('Updating job status')
        multijob.update()
        Printer.dyn(multijob.get_status_line())

        # if some jobs changed status add new line to dynamic output remains
        jobs_changed = multijob.get_all(status=JobState.COMPLETED)
        if jobs_changed:
            Printer.out()
            Printer.separator()

        # get all jobs where was status update to COMPLETE state
        for job in jobs_changed:
            returncodes[job] = finish_pbs_job(job, arg_options.batch)

        if jobs_changed:
            Printer.separator()
            Printer.out()

        # after printing update status lets sleep for a bit
        if multijob.is_running():
            time.sleep(5)

    Printer.out(multijob.get_status_line())
    Printer.out('All jobs finished')

    # get max return code or number 2 if there are no returncodes
    returncode = max(returncodes.values()) if returncodes else 2
    sys.exit(returncode)
Ejemplo n.º 25
0
    def compare(self, reference_filepath, other_filepath, **kwargs):
        """
        Method can do anything as long as int value is returned
        :param reference_filepath:
        :param other_filepath:
        :param kwargs:
        :return:
        """

        regex = kwargs.get('regex', None)
        substr = kwargs.get('substr', None)

        if regex is None and substr is None:
            self.output.write(
                "Invalid check rule! Specify either 'regex' or 'substr' keyword."
            )
            return 1

        # convert to str or compile to regex
        if substr is not None:
            substr = str(substr)
        if regex is not None:
            regex = re.compile(str(regex))

        # for regex comparison we ignore reference file
        # read job_output.log
        content = IO.read(Paths.abspath(other_filepath))

        # substr find
        if substr:
            found = False
            for line in content.splitlines():
                if line.find(substr) != -1:
                    found = True
                    self.output.write(
                        '[OK] Found substr "{substr}" in line:'.format(
                            **locals()))
                    self.output.write('    ' + line)
                    break
            if not found:
                self.output.write(
                    '[ERROR] Could not find substr "{substr}":'.format(
                        **locals()))
                return 1

        # regex match
        if regex:
            found = False
            for line in content.splitlines():
                if regex.findall(line):
                    found = True
                    self.output.write(
                        '[OK] Found regex "{regex.pattern}" in line:'.format(
                            **locals()))
                    self.output.write('    ' + line)
                    break
            if not found:
                self.output.write(
                    '[ERROR] Could not find regex {regex.pattern}:'.format(
                        **locals()))
                return 1

        # self.output.write("In case of emergency,")
        # self.output.write("    you can provide details on what went wrong")
        # self.output.write("    using self.output.write method")
        # self.output.write("")
        # self.output.write("Error while comparing files \n{} \n{}"
        #                   .format(reference_filepath, other_filepath))

        # must return return-code!
        return 0