def run_butler(
butler, test_cases, output_xml, log_file=None, log_level=logging.DEBUG, force_distribute=None, force_chunk=None
):
"""
Run a Butler instance.
:param butler: Butler instance
:return: exit code
:rtype: int
"""
started_at = time.time()
if isinstance(force_distribute, bool):
butler.force_distribute = force_distribute
if isinstance(force_chunk, bool):
butler.force_chunk = force_chunk
cmd = butler.to_cmd()
if not os.path.exists(butler.output_dir):
os.mkdir(butler.output_dir)
b_log = os.path.join(butler.output_dir, "butler.log")
# b_err = os.path.join(butler.output_dir, 'butler.stderr')
b_out = os.path.join(butler.output_dir, "butler.stdout")
setup_log(log, level=log_level, file_name=b_log)
# Butler stdout log
str_formatter = "[%(levelname)s] %(message)s"
setup_log(slog, level=log_level, file_name=b_out, log_filter=StdOutStatusLogFilter(), str_formatter=str_formatter)
if log_file is not None:
if isinstance(log_level, str):
log_levels_d = {attr: getattr(logging, attr) for attr in LOG_LEVELS}
level = log_levels_d[log_level]
else:
level = logging.DEBUG
setup_log(log, level=level, file_name=log_file)
# log.debug(pprint.pformat(butler.__dict__))
slog.info("Running butler with test id {i}".format(i=butler.job_id))
slog.info("Running cmd '{c}'".format(c=cmd))
def _to_p(file_name):
return os.path.join(butler.output_dir, file_name)
# pbsmrtpipe stdout and stderr
stdout = _to_p("job.stdout")
stderr = _to_p("job.stderr")
with open(stdout, "w") as stdout_fh:
with open(stderr, "w") as stderr_fh:
rcode, stdout, stderr, run_time = run_command_async(cmd, stdout_fh, stderr_fh)
rmessage = "was successful" if rcode == 0 else " failed"
msg = "pbsmrtpipe command {m} ({s:.2f} sec) exit code {e}.'".format(e=rcode, s=run_time, m=rmessage)
if rcode != 0:
slog.error(msg)
log.error(str(stderr))
else:
slog.info(msg)
if test_cases is not None:
trcode = run_butler_tests(test_cases, butler.output_dir, output_xml, butler.job_id)
else:
trcode = 0
run_time = time.time() - started_at
slog.info("Exiting testkit runner in {s:.2f} sec.".format(s=run_time))
# Was butler successful + all the tests pass
return rcode | trcode
def run_butler(butler, test_cases, output_xml,
log_file=None,
log_level=logging.DEBUG,
force_distribute=None,
force_chunk=None):
"""
Run a Butler instance.
:param butler: Butler instance
:return: exit code
:rtype: int
"""
started_at = time.time()
if isinstance(force_distribute, bool):
butler.force_distribute = force_distribute
if isinstance(force_chunk, bool):
butler.force_chunk = force_chunk
cmd = butler.to_cmd()
if not os.path.exists(butler.output_dir):
os.mkdir(butler.output_dir)
b_log = os.path.join(butler.output_dir, 'butler.log')
# b_err = os.path.join(butler.output_dir, 'butler.stderr')
b_out = os.path.join(butler.output_dir, 'butler.stdout')
setup_log(log, level=log_level, file_name=b_log)
# Butler stdout log
str_formatter = '[%(levelname)s] %(message)s'
setup_log(slog, level=log_level,
file_name=b_out,
log_filter=StdOutStatusLogFilter(),
str_formatter=str_formatter)
if log_file is not None:
if isinstance(log_level, str):
log_levels_d = {attr: getattr(logging, attr) for attr in LOG_LEVELS}
level = log_levels_d[log_level]
else:
level = logging.DEBUG
setup_log(log, level=level, file_name=log_file)
# log.debug(pprint.pformat(butler.__dict__))
slog.info("Running butler with test id {i}".format(i=butler.job_id))
slog.info("Running cmd '{c}'".format(c=cmd))
def _to_p(file_name):
return os.path.join(butler.output_dir, file_name)
# pbsmrtpipe stdout and stderr
stdout = _to_p('job.stdout')
stderr = _to_p('job.stderr')
with open(stdout, 'w') as stdout_fh:
with open(stderr, 'w') as stderr_fh:
rcode, stdout, stderr, run_time = run_command_async(cmd, stdout_fh, stderr_fh)
rmessage = "was successful" if rcode == 0 else " failed"
msg = "pbsmrtpipe command {m} ({s:.2f} sec) exit code {e}.'".format(e=rcode, s=run_time, m=rmessage)
if rcode != 0:
slog.error(msg)
log.error(str(stderr))
else:
slog.info(msg)
if test_cases is not None:
trcode = run_butler_tests(test_cases, butler.output_dir, output_xml, butler.job_id)
else:
trcode = 0
run_time = time.time() - started_at
slog.info("Exiting testkit runner in {s:.2f} sec.".format(s=run_time))
# Was butler successful + all the tests pass
return rcode | trcode
def run_butler(butler, test_cases, output_xml,
log_file,
log_level=logging.DEBUG,
force_distribute=None,
force_chunk=None,
ignore_test_failures=False):
"""
Run a Butler instance.
:param butler: Butler instance
:return: exit code
:rtype: int
"""
started_at = time.time()
if isinstance(force_distribute, bool):
butler.force_distribute = force_distribute
if isinstance(force_chunk, bool):
butler.force_chunk = force_chunk
cmd = butler.to_cmd()
if not os.path.exists(butler.output_dir):
os.mkdir(butler.output_dir)
if isinstance(log_level, str):
log_levels_d = {attr: getattr(logging, attr) for attr in LOG_LEVELS}
console_level = log_levels_d[log_level]
else:
console_level = log_level
# Always Write the log will full debug mode.
setup_console_and_file_logger(console_level, Constants.LOG_FMT_SIMPLE, log_file, logging.DEBUG, Constants.LOG_FMT_STD)
slog.info("completed setting up log to console and {f}".format(f=log_file))
slog.info("Running butler with test id {i}".format(i=butler.job_id))
slog.info("Running cmd '{c}'".format(c=cmd))
def _to_p(file_name):
return os.path.join(butler.output_dir, file_name)
# pbsmrtpipe stdout and stderr
stdout = _to_p('job.stdout')
stderr = _to_p('job.stderr')
with open(stdout, 'w') as stdout_fh:
with open(stderr, 'w') as stderr_fh:
rcode, stdout, stderr, run_time = run_command_async(cmd, stdout_fh, stderr_fh)
rmessage = "was successful" if rcode == 0 else " failed"
msg = "pbsmrtpipe command {m} ({s:.2f} sec) exit code {e}.'".format(e=rcode, s=run_time, m=rmessage)
if rcode != 0:
slog.error(msg)
log.error(str(stderr))
else:
slog.info(msg)
if test_cases is not None:
slog.info("Running in test-only mode")
trcode = run_butler_tests(test_cases, butler.output_dir, output_xml, butler.job_id)
else:
trcode = 0
run_time = time.time() - started_at
slog.info("Exiting testkit runner in {s:.2f} sec.".format(s=run_time))
# Was butler successful + all the tests pass
if ignore_test_failures:
return rcode
return rcode | trcode
def run_butler(butler,
test_cases,
output_xml,
log_file,
log_level=logging.DEBUG,
force_distribute=None,
force_chunk=None,
ignore_test_failures=False):
"""
Run a Butler instance.
:param butler: Butler instance
:return: exit code
:rtype: int
"""
started_at = time.time()
if isinstance(force_distribute, bool):
butler.force_distribute = force_distribute
if isinstance(force_chunk, bool):
butler.force_chunk = force_chunk
cmd = butler.to_cmd()
if not os.path.exists(butler.output_dir):
os.mkdir(butler.output_dir)
if isinstance(log_level, str):
log_levels_d = {attr: getattr(logging, attr) for attr in LOG_LEVELS}
console_level = log_levels_d[log_level]
else:
console_level = log_level
# Always Write the log will full debug mode.
setup_console_and_file_logger(console_level, Constants.LOG_FMT_SIMPLE,
log_file, logging.DEBUG,
Constants.LOG_FMT_STD)
slog.info("completed setting up log to console and {f}".format(f=log_file))
slog.info("Running butler with test id {i}".format(i=butler.job_id))
slog.info("Running cmd '{c}'".format(c=cmd))
def _to_p(file_name):
return os.path.join(butler.output_dir, file_name)
# pbsmrtpipe stdout and stderr
stdout = _to_p('job.stdout')
stderr = _to_p('job.stderr')
with open(stdout, 'w') as stdout_fh:
with open(stderr, 'w') as stderr_fh:
rcode, stdout, stderr, run_time = run_command_async(
cmd, stdout_fh, stderr_fh)
rmessage = "was successful" if rcode == 0 else " failed"
msg = "pbsmrtpipe command {m} ({s:.2f} sec) exit code {e}.'".format(
e=rcode, s=run_time, m=rmessage)
if rcode != 0:
slog.error(msg)
log.error(str(stderr))
else:
slog.info(msg)
if test_cases is not None:
slog.info("Running in test-only mode")
trcode = run_butler_tests(test_cases, butler.output_dir,
output_xml, butler.job_id)
else:
trcode = 0
run_time = time.time() - started_at
slog.info("Exiting testkit runner in {s:.2f} sec.".format(s=run_time))
# Was butler successful + all the tests pass
if ignore_test_failures:
return rcode
return rcode | trcode