def _pbsmrtipe_setup_log(alog, **kwargs):
"""Setup stdout log. pbsmrtpipe will setup pbsmrtpipe.log, master.log
This should only emit 'status.*' messages.
"""
str_formatter = '[%(levelname)s] %(asctime)-15s %(message)s'
level = kwargs.get('level', logging.INFO)
setup_log(alog,
level=level,
file_name=None,
log_filter=StdOutStatusLogFilter(),
str_formatter=str_formatter)
slog.info("Starting pbsmrtpipe v{v}".format(v=pbsmrtpipe.get_version()))
def _pbsmrtipe_setup_log(alog, **kwargs):
"""Setup stdout log. pbsmrtpipe will setup pbsmrtpipe.log, master.log
This should only emit 'status.*' messages.
"""
str_formatter = '[%(levelname)s] %(asctime)-15s %(message)s'
level = kwargs.get('level', logging.INFO)
setup_log(alog,
level=level,
file_name=None,
log_filter=StdOutStatusLogFilter(),
str_formatter=str_formatter)
slog.info("Starting pbsmrtpipe v{v}".format(v=pbsmrtpipe.get_version()))
def run_fofn(butler_cfg_fofn, log_file=None):
"""
Run fofn of butler.cfg files
:param butler_cfg_fofn:
"""
# list of absolute file paths
butler_cfgs = _parse_file(butler_cfg_fofn)
if log_file is not None:
setup_log(log, level=logging.DEBUG, file_name=log_file)
for butler_cfg in butler_cfgs:
cmd = _to_cmd(_EXE, butler_cfg)
dir_name = os.path.dirname(butler_cfg)
with lcd(dir_name):
log.info(cmd)
output = local(cmd)
log.info(output)
print output
def _pbsmrtipe_setup_log(alog, **kwargs):
"""Setup stdout log. pbsmrtpipe will setup pbsmrtpipe.log, master.log
This should only emit 'status.*' messages.
"""
# This is a essentially just a bootstrapping step before the job-dir/logs
# can be created and proper log files (pbsmrtpipe.log, master.log) will
# be setup for this to work with the new global dict setup model would have to
# extended to support adding a custom filter.
str_formatter = '%(message)s'
level = kwargs.get('level', logging.INFO)
setup_log(alog,
level=level,
file_name=None,
log_filter=StdOutStatusLogFilter(),
str_formatter=str_formatter)
slog.info("Starting pbsmrtpipe v{v}".format(v=pbsmrtpipe.get_version()))
def _pbsmrtipe_setup_log(alog, **kwargs):
"""Setup stdout log. pbsmrtpipe will setup pbsmrtpipe.log, master.log
This should only emit 'status.*' messages.
"""
# This is a essentially just a bootstrapping step before the job-dir/logs
# can be created and proper log files (pbsmrtpipe.log, master.log) will
# be setup for this to work with the new global dict setup model would have to
# extended to support adding a custom filter.
str_formatter = '%(message)s'
level = kwargs.get('level', logging.INFO)
setup_log(alog,
level=level,
file_name=None,
log_filter=StdOutStatusLogFilter(),
str_formatter=str_formatter)
slog.info("Starting pbsmrtpipe v{v}".format(v=pbsmrtpipe.get_version()))
def run_butler(
butler, test_cases, output_xml, log_file=None, log_level=logging.DEBUG, force_distribute=None, force_chunk=None
):
"""
Run a Butler instance.
:param butler: Butler instance
:return: exit code
:rtype: int
"""
started_at = time.time()
if isinstance(force_distribute, bool):
butler.force_distribute = force_distribute
if isinstance(force_chunk, bool):
butler.force_chunk = force_chunk
cmd = butler.to_cmd()
if not os.path.exists(butler.output_dir):
os.mkdir(butler.output_dir)
b_log = os.path.join(butler.output_dir, "butler.log")
# b_err = os.path.join(butler.output_dir, 'butler.stderr')
b_out = os.path.join(butler.output_dir, "butler.stdout")
setup_log(log, level=log_level, file_name=b_log)
# Butler stdout log
str_formatter = "[%(levelname)s] %(message)s"
setup_log(slog, level=log_level, file_name=b_out, log_filter=StdOutStatusLogFilter(), str_formatter=str_formatter)
if log_file is not None:
if isinstance(log_level, str):
log_levels_d = {attr: getattr(logging, attr) for attr in LOG_LEVELS}
level = log_levels_d[log_level]
else:
level = logging.DEBUG
setup_log(log, level=level, file_name=log_file)
# log.debug(pprint.pformat(butler.__dict__))
slog.info("Running butler with test id {i}".format(i=butler.job_id))
slog.info("Running cmd '{c}'".format(c=cmd))
def _to_p(file_name):
return os.path.join(butler.output_dir, file_name)
# pbsmrtpipe stdout and stderr
stdout = _to_p("job.stdout")
stderr = _to_p("job.stderr")
with open(stdout, "w") as stdout_fh:
with open(stderr, "w") as stderr_fh:
rcode, stdout, stderr, run_time = run_command_async(cmd, stdout_fh, stderr_fh)
rmessage = "was successful" if rcode == 0 else " failed"
msg = "pbsmrtpipe command {m} ({s:.2f} sec) exit code {e}.'".format(e=rcode, s=run_time, m=rmessage)
if rcode != 0:
slog.error(msg)
log.error(str(stderr))
else:
slog.info(msg)
if test_cases is not None:
trcode = run_butler_tests(test_cases, butler.output_dir, output_xml, butler.job_id)
else:
trcode = 0
run_time = time.time() - started_at
slog.info("Exiting testkit runner in {s:.2f} sec.".format(s=run_time))
# Was butler successful + all the tests pass
return rcode | trcode
def setup_minimal_log(alog, **kwargs):
kwargs["str_formatter"] = "%(message)s"
setup_log(alog, **kwargs)
def run_butler(butler, test_cases, output_xml,
log_file=None,
log_level=logging.DEBUG,
force_distribute=None,
force_chunk=None):
"""
Run a Butler instance.
:param butler: Butler instance
:return: exit code
:rtype: int
"""
started_at = time.time()
if isinstance(force_distribute, bool):
butler.force_distribute = force_distribute
if isinstance(force_chunk, bool):
butler.force_chunk = force_chunk
cmd = butler.to_cmd()
if not os.path.exists(butler.output_dir):
os.mkdir(butler.output_dir)
b_log = os.path.join(butler.output_dir, 'butler.log')
# b_err = os.path.join(butler.output_dir, 'butler.stderr')
b_out = os.path.join(butler.output_dir, 'butler.stdout')
setup_log(log, level=log_level, file_name=b_log)
# Butler stdout log
str_formatter = '[%(levelname)s] %(message)s'
setup_log(slog, level=log_level,
file_name=b_out,
log_filter=StdOutStatusLogFilter(),
str_formatter=str_formatter)
if log_file is not None:
if isinstance(log_level, str):
log_levels_d = {attr: getattr(logging, attr) for attr in LOG_LEVELS}
level = log_levels_d[log_level]
else:
level = logging.DEBUG
setup_log(log, level=level, file_name=log_file)
# log.debug(pprint.pformat(butler.__dict__))
slog.info("Running butler with test id {i}".format(i=butler.job_id))
slog.info("Running cmd '{c}'".format(c=cmd))
def _to_p(file_name):
return os.path.join(butler.output_dir, file_name)
# pbsmrtpipe stdout and stderr
stdout = _to_p('job.stdout')
stderr = _to_p('job.stderr')
with open(stdout, 'w') as stdout_fh:
with open(stderr, 'w') as stderr_fh:
rcode, stdout, stderr, run_time = run_command_async(cmd, stdout_fh, stderr_fh)
rmessage = "was successful" if rcode == 0 else " failed"
msg = "pbsmrtpipe command {m} ({s:.2f} sec) exit code {e}.'".format(e=rcode, s=run_time, m=rmessage)
if rcode != 0:
slog.error(msg)
log.error(str(stderr))
else:
slog.info(msg)
if test_cases is not None:
trcode = run_butler_tests(test_cases, butler.output_dir, output_xml, butler.job_id)
else:
trcode = 0
run_time = time.time() - started_at
slog.info("Exiting testkit runner in {s:.2f} sec.".format(s=run_time))
# Was butler successful + all the tests pass
return rcode | trcode
def setup_minimal_log(alog, **kwargs):
kwargs['str_formatter'] = '%(message)s'
setup_log(alog, **kwargs)
