Example #1
0
def run_butler(
    butler, test_cases, output_xml, log_file=None, log_level=logging.DEBUG, force_distribute=None, force_chunk=None
):
    """
    Run a Butler instance.

    :param butler: Butler instance
    :return: exit code


    :rtype: int
    """
    started_at = time.time()

    if isinstance(force_distribute, bool):
        butler.force_distribute = force_distribute

    if isinstance(force_chunk, bool):
        butler.force_chunk = force_chunk

    cmd = butler.to_cmd()

    if not os.path.exists(butler.output_dir):
        os.mkdir(butler.output_dir)

    b_log = os.path.join(butler.output_dir, "butler.log")
    # b_err = os.path.join(butler.output_dir, 'butler.stderr')
    b_out = os.path.join(butler.output_dir, "butler.stdout")

    setup_log(log, level=log_level, file_name=b_log)

    # Butler stdout log
    str_formatter = "[%(levelname)s] %(message)s"
    setup_log(slog, level=log_level, file_name=b_out, log_filter=StdOutStatusLogFilter(), str_formatter=str_formatter)

    if log_file is not None:
        if isinstance(log_level, str):
            log_levels_d = {attr: getattr(logging, attr) for attr in LOG_LEVELS}
            level = log_levels_d[log_level]
        else:
            level = logging.DEBUG
        setup_log(log, level=level, file_name=log_file)

    # log.debug(pprint.pformat(butler.__dict__))
    slog.info("Running butler with test id {i}".format(i=butler.job_id))
    slog.info("Running cmd '{c}'".format(c=cmd))

    def _to_p(file_name):
        return os.path.join(butler.output_dir, file_name)

    # pbsmrtpipe stdout and stderr
    stdout = _to_p("job.stdout")
    stderr = _to_p("job.stderr")

    with open(stdout, "w") as stdout_fh:
        with open(stderr, "w") as stderr_fh:

            rcode, stdout, stderr, run_time = run_command_async(cmd, stdout_fh, stderr_fh)

            rmessage = "was successful" if rcode == 0 else " failed"
            msg = "pbsmrtpipe command {m} ({s:.2f} sec) exit code {e}.'".format(e=rcode, s=run_time, m=rmessage)
            if rcode != 0:
                slog.error(msg)
                log.error(str(stderr))
            else:
                slog.info(msg)

        if test_cases is not None:
            trcode = run_butler_tests(test_cases, butler.output_dir, output_xml, butler.job_id)
        else:
            trcode = 0

    run_time = time.time() - started_at
    slog.info("Exiting testkit runner in {s:.2f} sec.".format(s=run_time))

    # Was butler successful + all the tests pass
    return rcode | trcode
Example #2
0
def run_butler(butler, test_cases, output_xml,
               log_file=None,
               log_level=logging.DEBUG,
               force_distribute=None,
               force_chunk=None):
    """
    Run a Butler instance.

    :param butler: Butler instance
    :return: exit code


    :rtype: int
    """
    started_at = time.time()

    if isinstance(force_distribute, bool):
        butler.force_distribute = force_distribute

    if isinstance(force_chunk, bool):
        butler.force_chunk = force_chunk

    cmd = butler.to_cmd()

    if not os.path.exists(butler.output_dir):
        os.mkdir(butler.output_dir)

    b_log = os.path.join(butler.output_dir, 'butler.log')
    # b_err = os.path.join(butler.output_dir, 'butler.stderr')
    b_out = os.path.join(butler.output_dir, 'butler.stdout')

    setup_log(log, level=log_level, file_name=b_log)

    # Butler stdout log
    str_formatter = '[%(levelname)s] %(message)s'
    setup_log(slog, level=log_level,
              file_name=b_out,
              log_filter=StdOutStatusLogFilter(),
              str_formatter=str_formatter)

    if log_file is not None:
        if isinstance(log_level, str):
            log_levels_d = {attr: getattr(logging, attr) for attr in LOG_LEVELS}
            level = log_levels_d[log_level]
        else:
            level = logging.DEBUG
        setup_log(log, level=level, file_name=log_file)

    # log.debug(pprint.pformat(butler.__dict__))
    slog.info("Running butler with test id {i}".format(i=butler.job_id))
    slog.info("Running cmd '{c}'".format(c=cmd))

    def _to_p(file_name):
        return os.path.join(butler.output_dir, file_name)

    # pbsmrtpipe stdout and stderr
    stdout = _to_p('job.stdout')
    stderr = _to_p('job.stderr')

    with open(stdout, 'w') as stdout_fh:
        with open(stderr, 'w') as stderr_fh:

            rcode, stdout, stderr, run_time = run_command_async(cmd, stdout_fh, stderr_fh)

            rmessage = "was successful" if rcode == 0 else " failed"
            msg = "pbsmrtpipe command {m} ({s:.2f} sec) exit code {e}.'".format(e=rcode, s=run_time, m=rmessage)
            if rcode != 0:
                slog.error(msg)
                log.error(str(stderr))
            else:
                slog.info(msg)

        if test_cases is not None:
            trcode = run_butler_tests(test_cases, butler.output_dir, output_xml, butler.job_id)
        else:
            trcode = 0

    run_time = time.time() - started_at
    slog.info("Exiting testkit runner in {s:.2f} sec.".format(s=run_time))

    # Was butler successful + all the tests pass
    return rcode | trcode
Example #3
0
def run_butler(butler, test_cases, output_xml,
               log_file,
               log_level=logging.DEBUG,
               force_distribute=None,
               force_chunk=None,
               ignore_test_failures=False):
    """
    Run a Butler instance.

    :param butler: Butler instance
    :return: exit code


    :rtype: int
    """
    started_at = time.time()

    if isinstance(force_distribute, bool):
        butler.force_distribute = force_distribute

    if isinstance(force_chunk, bool):
        butler.force_chunk = force_chunk

    cmd = butler.to_cmd()

    if not os.path.exists(butler.output_dir):
        os.mkdir(butler.output_dir)

    if isinstance(log_level, str):
        log_levels_d = {attr: getattr(logging, attr) for attr in LOG_LEVELS}
        console_level = log_levels_d[log_level]
    else:
        console_level = log_level

    # Always Write the log will full debug mode.
    setup_console_and_file_logger(console_level, Constants.LOG_FMT_SIMPLE, log_file, logging.DEBUG, Constants.LOG_FMT_STD)

    slog.info("completed setting up log to console and {f}".format(f=log_file))
    slog.info("Running butler with test id {i}".format(i=butler.job_id))
    slog.info("Running cmd '{c}'".format(c=cmd))

    def _to_p(file_name):
        return os.path.join(butler.output_dir, file_name)

    # pbsmrtpipe stdout and stderr
    stdout = _to_p('job.stdout')
    stderr = _to_p('job.stderr')

    with open(stdout, 'w') as stdout_fh:
        with open(stderr, 'w') as stderr_fh:

            rcode, stdout, stderr, run_time = run_command_async(cmd, stdout_fh, stderr_fh)

            rmessage = "was successful" if rcode == 0 else " failed"
            msg = "pbsmrtpipe command {m} ({s:.2f} sec) exit code {e}.'".format(e=rcode, s=run_time, m=rmessage)
            if rcode != 0:
                slog.error(msg)
                log.error(str(stderr))
            else:
                slog.info(msg)

        if test_cases is not None:
            slog.info("Running in test-only mode")
            trcode = run_butler_tests(test_cases, butler.output_dir, output_xml, butler.job_id)
        else:
            trcode = 0

    run_time = time.time() - started_at
    slog.info("Exiting testkit runner in {s:.2f} sec.".format(s=run_time))

    # Was butler successful + all the tests pass
    if ignore_test_failures:
        return rcode
    return rcode | trcode
Example #4
0
def run_butler(butler,
               test_cases,
               output_xml,
               log_file,
               log_level=logging.DEBUG,
               force_distribute=None,
               force_chunk=None,
               ignore_test_failures=False):
    """
    Run a Butler instance.

    :param butler: Butler instance
    :return: exit code


    :rtype: int
    """
    started_at = time.time()

    if isinstance(force_distribute, bool):
        butler.force_distribute = force_distribute

    if isinstance(force_chunk, bool):
        butler.force_chunk = force_chunk

    cmd = butler.to_cmd()

    if not os.path.exists(butler.output_dir):
        os.mkdir(butler.output_dir)

    if isinstance(log_level, str):
        log_levels_d = {attr: getattr(logging, attr) for attr in LOG_LEVELS}
        console_level = log_levels_d[log_level]
    else:
        console_level = log_level

    # Always Write the log will full debug mode.
    setup_console_and_file_logger(console_level, Constants.LOG_FMT_SIMPLE,
                                  log_file, logging.DEBUG,
                                  Constants.LOG_FMT_STD)

    slog.info("completed setting up log to console and {f}".format(f=log_file))
    slog.info("Running butler with test id {i}".format(i=butler.job_id))
    slog.info("Running cmd '{c}'".format(c=cmd))

    def _to_p(file_name):
        return os.path.join(butler.output_dir, file_name)

    # pbsmrtpipe stdout and stderr
    stdout = _to_p('job.stdout')
    stderr = _to_p('job.stderr')

    with open(stdout, 'w') as stdout_fh:
        with open(stderr, 'w') as stderr_fh:

            rcode, stdout, stderr, run_time = run_command_async(
                cmd, stdout_fh, stderr_fh)

            rmessage = "was successful" if rcode == 0 else " failed"
            msg = "pbsmrtpipe command {m} ({s:.2f} sec) exit code {e}.'".format(
                e=rcode, s=run_time, m=rmessage)
            if rcode != 0:
                slog.error(msg)
                log.error(str(stderr))
            else:
                slog.info(msg)

        if test_cases is not None:
            slog.info("Running in test-only mode")
            trcode = run_butler_tests(test_cases, butler.output_dir,
                                      output_xml, butler.job_id)
        else:
            trcode = 0

    run_time = time.time() - started_at
    slog.info("Exiting testkit runner in {s:.2f} sec.".format(s=run_time))

    # Was butler successful + all the tests pass
    if ignore_test_failures:
        return rcode
    return rcode | trcode