def to_dict(self, id_parts=None):
version = pbcommand.get_version()
d = BaseReportElement.to_dict(self, id_parts=id_parts)
d['_version'] = version
d['_changelist'] = "UNKNOWN"
d['dataset_uuids'] = list(set(self._dataset_uuids))
return d
def to_dict(self, id_parts=None):
version = pbcommand.get_version()
d = BaseReportElement.to_dict(self, id_parts=id_parts)
d['_version'] = version
d['_changelist'] = "UNKNOWN"
d['dataset_uuids'] = list(set(self._dataset_uuids))
return d
def to_dict(self, id_parts=None):
version = pbcommand.get_version()
d = BaseReportElement.to_dict(self, id_parts=id_parts)
d["title"] = self.title
d["_version"] = version
d["_changelist"] = "UNKNOWN"
d["dataset_uuids"] = list(set(self._dataset_uuids))
return d
def to_dict(self):
comment = "Created by pbcommand v{v}".format(v=pbcommand.get_version())
tc = dict(input_files=self.input_files,
output_files=self.output_files,
task_type=self.TASK_TYPE_ID,
is_distributed=self.is_distributed,
tool_contract_id=self.task_id,
nproc=self.nproc,
resources=[r.to_dict() for r in self.resources],
options=self.options,
_comment=comment)
return tc
def to_dict(self):
comment = "Created by pbcommand v{v}".format(v=pbcommand.get_version())
tc = dict(input_files=self.input_files,
output_files=self.output_files,
task_type=self.TASK_TYPE_ID,
is_distributed=self.is_distributed,
tool_contract_id=self.task_id,
nproc=self.nproc,
resources=self.resources,
options=self.options,
_comment=comment)
return tc
def to_dict(self, id_parts=None):
_d = dict(v=pbcommand.get_version(),
t=datetime.datetime.now().isoformat())
d = BaseReportElement.to_dict(self, id_parts=id_parts)
d['_comment'] = "Generated with pbcommand version {v} at {t}".format(**_d)
# Required in 1.0.0 of the spec
d['uuid'] = self.uuid
d['title'] = self.title
d['version'] = PB_REPORT_SCHEMA_VERSION
d['dataset_uuids'] = list(set(self.dataset_uuids))
return d
def to_dict(self, id_parts=None):
_d = dict(v=pbcommand.get_version(),
t=datetime.datetime.now().isoformat())
d = BaseReportElement.to_dict(self, id_parts=id_parts)
d['_comment'] = "Generated with pbcommand version {v} at {t}".format(
**_d)
# Required in 1.0.0 of the spec
d['uuid'] = self.uuid
d['title'] = self.title
d['version'] = PB_REPORT_SCHEMA_VERSION
d['dataset_uuids'] = list(set(self.dataset_uuids))
return d
def _pacbio_main_runner(alog, setup_log_func, func, *args, **kwargs):
"""
Runs a general func and logs results. The return type is expected to be an (int) return code.
:param alog: a log instance
:param func: a cli exe func, must return an int exit code. func(args) => Int, where args is parsed from p.parse_args()
:param args: parsed args from parser
:return: Exit code of callable func
:rtype: int
"""
started_at = time.time()
pargs = args[0]
level = logging.INFO
# Assuming that the log_level might not be an added option.
if hasattr(pargs, 'log_level'):
level = logging.getLevelName(pargs.log_level)
log_options = dict(level=level)
# The Setup log func must adhere to the pbcommand.utils.setup_log func
# signature
setup_log_func(alog, **log_options)
alog.info("Using pbcommand v{v}".format(v=pbcommand.get_version()))
try:
# the code in func should catch any exceptions. The try/catch
# here is a fail safe to make sure the program doesn't fail
# and the makes sure the exit code is logged.
return_code = func(*args, **kwargs)
run_time = time.time() - started_at
except Exception as e:
run_time = time.time() - started_at
alog.error(e, exc_info=True)
traceback.print_exc(sys.stderr)
# We should have a standard map of exit codes to Int
if isinstance(e, IOError):
return_code = 1
else:
return_code = 2
_d = dict(r=return_code, s=run_time)
alog.info("exiting with return code {r} in {s:.2f} sec.".format(**_d))
return return_code
def _w(args):
# FIXME.
setup_log(log, level=logging.DEBUG)
log.info("Loading pbcommand {v}".format(v=pbcommand.get_version()))
log.info("Registry {r}".format(r=registry))
log.info("loading RTC from {i}".format(i=args.rtc_path))
rtc = load_resolved_tool_contract_from(args.rtc_path)
id_funcs = {t.task.task_id:func for t, func in registry.rtc_runners.iteritems()}
func = id_funcs.get(rtc.task.task_id, None)
if func is None:
sys.stderr.write("ERROR. Unknown tool contract id {x}".format(x=rtc.task.task_id))
return -1
else:
log.info("Running id:{i} Resolved Tool Contract {r}".format(r=rtc, i=rtc.task.task_id))
log.info("Runner func {f}".format(f=func))
exit_code = func(rtc)
log.info("Completed running {r} exitcode {e}".format(r=rtc, e=exit_code))
return exit_code
def _w(args):
started_at = time.time()
def run_time():
return time.time() - started_at
def exit_msg(rcode_):
return "Completed running {r} exitcode {e} in {t:.2f} sec.".format(
r=rtc, e=rcode_, t=run_time())
level = get_parsed_args_log_level(args)
setup_logger(None, level=level)
log.info("Loading pbcommand {v}".format(v=pbcommand.get_version()))
log.info("Registry {r}".format(r=registry))
log.info("Setting log-level to {d}".format(d=level))
log.debug("args {a}".format(a=args))
log.info("loading RTC from {i}".format(i=args.rtc_path))
rtc = load_resolved_tool_contract_from(args.rtc_path)
id_funcs = {
t.task.task_id: func
for t, func in registry.rtc_runners.iteritems()
}
func = id_funcs.get(rtc.task.task_id, None)
if func is None:
rcode = 1
log.error(
"Unknown tool contract id '{x}' Registered TC ids {i}".format(
x=rtc.task.task_id, i=id_funcs.keys()))
log.error(exit_msg(rcode))
return rcode
else:
log.info("Running id:{i} Resolved Tool Contract {r}".format(
r=rtc, i=rtc.task.task_id))
log.info("Runner func {f}".format(f=func))
exit_code = func(rtc)
if exit_code == 0:
log.info(exit_msg(exit_code))
else:
log.error(exit_msg(exit_code))
return exit_code
def _w(args):
started_at = time.time()
def run_time():
return time.time() - started_at
def exit_msg(rcode_):
return "Completed running {r} exitcode {e} in {t:.2f} sec.".format(r=rtc, e=rcode_, t=run_time())
level = get_parsed_args_log_level(args)
setup_logger(None, level=level)
log.info("Loading pbcommand {v}".format(v=pbcommand.get_version()))
log.info("Registry {r}".format(r=registry))
log.info("Setting log-level to {d}".format(d=level))
log.debug("args {a}".format(a=args))
log.info("loading RTC from {i}".format(i=args.rtc_path))
rtc = load_resolved_tool_contract_from(args.rtc_path)
id_funcs = {t.task.task_id: func for t, func in registry.rtc_runners.iteritems()}
func = id_funcs.get(rtc.task.task_id, None)
if func is None:
rcode = 1
log.error("Unknown tool contract id '{x}' Registered TC ids {i}".format(x=rtc.task.task_id, i=id_funcs.keys()))
log.error(exit_msg(rcode))
return rcode
else:
log.info("Running id:{i} Resolved Tool Contract {r}".format(r=rtc, i=rtc.task.task_id))
log.info("Runner func {f}".format(f=func))
exit_code = func(rtc)
if exit_code == 0:
log.info(exit_msg(exit_code))
else:
log.error(exit_msg(exit_code))
return exit_code
def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs):
"""
Runs a general func and logs results. The return type is expected to be an (int) return code.
:param alog: a log instance
:param func: a cli exe func, must return an int exit code. func(args) => Int, where args is parsed from p.parse_args()
:param args: parsed args from parser
:param setup_log_func: F(alog, level=value, file_name=value, formatter=value) or None
:return: Exit code of callable func
:rtype: int
"""
started_at = time.time()
pargs = args[0]
# default logging level
level = logging.INFO
if 'level' in kwargs:
level = kwargs.pop('level')
else:
level = get_parsed_args_log_level(pargs)
# None will default to stdout
log_file = getattr(pargs, 'log_file', None)
# Currently, only support to stdout. More customization would require
# more required commandline options in base parser (e.g., --log-file,
# --log-formatter)
log_options = dict(level=level, file_name=log_file)
base_dir = os.getcwd()
dump_alarm_on_error = False
if "dump_alarm_on_error" in kwargs:
dump_alarm_on_error = kwargs.pop("dump_alarm_on_error")
is_cromwell_environment = bool(
os.environ.get("SMRT_PIPELINE_BUNDLE_DIR",
None)) and "cromwell-executions" in base_dir
dump_alarm_on_error = dump_alarm_on_error and is_cromwell_environment
# The Setup log func must adhere to the pbcommand.utils.setup_log func
# signature
# FIXME. This should use the more concrete F(file_name_or_name, level, formatter)
# signature of setup_logger
if setup_log_func is not None and alog is not None:
setup_log_func(alog, **log_options)
alog.info("Using pbcommand v{v}".format(v=pbcommand.get_version()))
alog.info(
"completed setting up logger with {f}".format(f=setup_log_func))
alog.info("log opts {d}".format(d=log_options))
if dump_alarm_on_error:
alog.info(
"This command appears to be running as part of a Cromwell workflow"
)
alog.info("Additional output files may be generated")
try:
# the code in func should catch any exceptions. The try/catch
# here is a fail safe to make sure the program doesn't fail
# and the makes sure the exit code is logged.
return_code = exe_main_func(*args, **kwargs)
run_time = time.time() - started_at
except Exception as e:
run_time = time.time() - started_at
if alog is not None:
alog.error(e, exc_info=True)
else:
traceback.print_exc(sys.stderr)
if dump_alarm_on_error:
PacBioAlarm.dump_error(file_name=os.path.join(
base_dir, "alarms.json"),
exception=e,
info="".join(traceback.format_exc()),
message=str(e),
name=e.__class__.__name__,
severity=logging.ERROR)
# We should have a standard map of exit codes to Int
if isinstance(e, IOError):
return_code = 1
else:
return_code = 2
_d = dict(r=return_code, s=run_time)
if is_cromwell_environment:
alog.info("Writing task report to task-report.json")
write_task_report(run_time, getattr(pargs, "nproc", 1), return_code)
if alog is not None:
alog.info("exiting with return code {r} in {s:.2f} sec.".format(**_d))
return return_code
#source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'pbcommand' copyright = u'2015-2016, Michael Kocher' author = u'Michael Kocher' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = pbcommand.get_version() # The full version, including alpha/beta/rc tags. release = pbcommand.get_version() # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y'
"""Common models for Tool Contract and Resolved Tool Contract
Author: Michael Kocher
"""
import abc
import pbcommand
from .common import TaskTypes, ResourceTypes
__version__ = pbcommand.get_version()
class MalformedToolContractError(ValueError):
pass
class MalformedResolvedToolContractError(ValueError):
pass
def _validate_type(value, type_or_types):
return isinstance(value, type_or_types)
def _validate_or_raise(value, type_or_types):
if not _validate_type(value, type_or_types):
_d = dict(x=value, t=type(value), s=type_or_types)
raise TypeError("Unsupported type for {x} {t}. Expected types {s}".format(**_d))
return value
"""Common models for Tool Contract and Resolved Tool Contract
Author: Michael Kocher
"""
import abc
from collections import OrderedDict
import types
import datetime
import pbcommand
from .common import TaskTypes, ResourceTypes, REGISTERED_FILE_TYPES
__version__ = pbcommand.get_version()
class MalformedToolContractError(ValueError):
pass
class MalformedResolvedToolContractError(ValueError):
pass
def _validate_type(value, type_or_types):
return isinstance(value, type_or_types)
def _validate_or_raise(value, type_or_types):
if not _validate_type(value, type_or_types):
#source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'pbcommand' copyright = u'2015-2017, Michael Kocher' author = u'Michael Kocher' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = pbcommand.get_version() # The full version, including alpha/beta/rc tags. release = pbcommand.get_version() # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y'
def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs):
"""
Runs a general func and logs results. The return type is expected to be an (int) return code.
:param alog: a log instance
:param func: a cli exe func, must return an int exit code. func(args) => Int, where args is parsed from p.parse_args()
:param args: parsed args from parser
:param setup_log_func: F(alog, level=value, file_name=value, formatter=value) or None
:return: Exit code of callable func
:rtype: int
"""
started_at = time.time()
pargs = args[0]
# default logging level
level = logging.INFO
if 'level' in kwargs:
level = kwargs.pop('level')
else:
level = get_parsed_args_log_level(pargs)
# None will default to stdout
log_file = getattr(pargs, 'log_file', None)
# Currently, only support to stdout. More customization would require
# more required commandline options in base parser (e.g., --log-file, --log-formatter)
log_options = dict(level=level, file_name=log_file)
# The Setup log func must adhere to the pbcommand.utils.setup_log func
# signature
# FIXME. This should use the more concrete F(file_name_or_name, level, formatter)
# signature of setup_logger
if setup_log_func is not None and alog is not None:
setup_log_func(alog, **log_options)
alog.info("Using pbcommand v{v}".format(v=pbcommand.get_version()))
alog.info(
"completed setting up logger with {f}".format(f=setup_log_func))
alog.info("log opts {d}".format(d=log_options))
try:
# the code in func should catch any exceptions. The try/catch
# here is a fail safe to make sure the program doesn't fail
# and the makes sure the exit code is logged.
return_code = exe_main_func(*args, **kwargs)
run_time = time.time() - started_at
except Exception as e:
run_time = time.time() - started_at
if alog is not None:
alog.error(e, exc_info=True)
else:
traceback.print_exc(sys.stderr)
# We should have a standard map of exit codes to Int
if isinstance(e, IOError):
return_code = 1
else:
return_code = 2
_d = dict(r=return_code, s=run_time)
if alog is not None:
alog.info("exiting with return code {r} in {s:.2f} sec.".format(**_d))
return return_code
def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs):
"""
Runs a general func and logs results. The return type is expected to be an (int) return code.
:param alog: a log instance
:param func: a cli exe func, must return an int exit code. func(args) => Int, where args is parsed from p.parse_args()
:param args: parsed args from parser
:param setup_log_func: F(alog, level=value, file_name=value, formatter=value) or None
:return: Exit code of callable func
:rtype: int
"""
started_at = time.time()
pargs = args[0]
# default logging level
level = logging.INFO
if 'level' in kwargs:
level = kwargs['level']
# If level is provided at the parser level, this will override all other
# settings. This is defined in pbcommand.common_options.add_log_level_option
if hasattr(pargs, 'log_level'):
level = logging.getLevelName(pargs.log_level)
# None will default to stdout
log_file = getattr(pargs, 'log_file', None)
# Currently, only support to stdout. More customization would require
# more required commandline options in base parser (e.g., --log-file, --log-formatter)
log_options = dict(level=level, file_name=log_file)
# The Setup log func must adhere to the pbcommand.utils.setup_log func
# signature
# FIXME. This should use the more concrete F(file_name_or_name, level, formatter)
# signature of setup_logger
if setup_log_func is not None and alog is not None:
setup_log_func(alog, **log_options)
alog.info("Using pbcommand v{v}".format(v=pbcommand.get_version()))
alog.info("completed setting up logger with {f}".format(f=setup_log_func))
alog.info("log opts {d}".format(d=log_options))
try:
# the code in func should catch any exceptions. The try/catch
# here is a fail safe to make sure the program doesn't fail
# and the makes sure the exit code is logged.
return_code = exe_main_func(*args, **kwargs)
run_time = time.time() - started_at
except Exception as e:
run_time = time.time() - started_at
if alog is not None:
alog.error(e, exc_info=True)
traceback.print_exc(sys.stderr)
# We should have a standard map of exit codes to Int
if isinstance(e, IOError):
return_code = 1
else:
return_code = 2
_d = dict(r=return_code, s=run_time)
if alog is not None:
alog.info("exiting with return code {r} in {s:.2f} sec.".format(**_d))
return return_code
