def to_dict(self, id_parts=None): version = pbcommand.get_version() d = BaseReportElement.to_dict(self, id_parts=id_parts) d['_version'] = version d['_changelist'] = "UNKNOWN" d['dataset_uuids'] = list(set(self._dataset_uuids)) return d
def to_dict(self, id_parts=None): version = pbcommand.get_version() d = BaseReportElement.to_dict(self, id_parts=id_parts) d["title"] = self.title d["_version"] = version d["_changelist"] = "UNKNOWN" d["dataset_uuids"] = list(set(self._dataset_uuids)) return d
def to_dict(self): comment = "Created by pbcommand v{v}".format(v=pbcommand.get_version()) tc = dict(input_files=self.input_files, output_files=self.output_files, task_type=self.TASK_TYPE_ID, is_distributed=self.is_distributed, tool_contract_id=self.task_id, nproc=self.nproc, resources=[r.to_dict() for r in self.resources], options=self.options, _comment=comment) return tc
def to_dict(self): comment = "Created by pbcommand v{v}".format(v=pbcommand.get_version()) tc = dict(input_files=self.input_files, output_files=self.output_files, task_type=self.TASK_TYPE_ID, is_distributed=self.is_distributed, tool_contract_id=self.task_id, nproc=self.nproc, resources=self.resources, options=self.options, _comment=comment) return tc
def to_dict(self, id_parts=None): _d = dict(v=pbcommand.get_version(), t=datetime.datetime.now().isoformat()) d = BaseReportElement.to_dict(self, id_parts=id_parts) d['_comment'] = "Generated with pbcommand version {v} at {t}".format(**_d) # Required in 1.0.0 of the spec d['uuid'] = self.uuid d['title'] = self.title d['version'] = PB_REPORT_SCHEMA_VERSION d['dataset_uuids'] = list(set(self.dataset_uuids)) return d
def to_dict(self, id_parts=None): _d = dict(v=pbcommand.get_version(), t=datetime.datetime.now().isoformat()) d = BaseReportElement.to_dict(self, id_parts=id_parts) d['_comment'] = "Generated with pbcommand version {v} at {t}".format( **_d) # Required in 1.0.0 of the spec d['uuid'] = self.uuid d['title'] = self.title d['version'] = PB_REPORT_SCHEMA_VERSION d['dataset_uuids'] = list(set(self.dataset_uuids)) return d
def _pacbio_main_runner(alog, setup_log_func, func, *args, **kwargs): """ Runs a general func and logs results. The return type is expected to be an (int) return code. :param alog: a log instance :param func: a cli exe func, must return an int exit code. func(args) => Int, where args is parsed from p.parse_args() :param args: parsed args from parser :return: Exit code of callable func :rtype: int """ started_at = time.time() pargs = args[0] level = logging.INFO # Assuming that the log_level might not be an added option. if hasattr(pargs, 'log_level'): level = logging.getLevelName(pargs.log_level) log_options = dict(level=level) # The Setup log func must adhere to the pbcommand.utils.setup_log func # signature setup_log_func(alog, **log_options) alog.info("Using pbcommand v{v}".format(v=pbcommand.get_version())) try: # the code in func should catch any exceptions. The try/catch # here is a fail safe to make sure the program doesn't fail # and the makes sure the exit code is logged. return_code = func(*args, **kwargs) run_time = time.time() - started_at except Exception as e: run_time = time.time() - started_at alog.error(e, exc_info=True) traceback.print_exc(sys.stderr) # We should have a standard map of exit codes to Int if isinstance(e, IOError): return_code = 1 else: return_code = 2 _d = dict(r=return_code, s=run_time) alog.info("exiting with return code {r} in {s:.2f} sec.".format(**_d)) return return_code
def _w(args): # FIXME. setup_log(log, level=logging.DEBUG) log.info("Loading pbcommand {v}".format(v=pbcommand.get_version())) log.info("Registry {r}".format(r=registry)) log.info("loading RTC from {i}".format(i=args.rtc_path)) rtc = load_resolved_tool_contract_from(args.rtc_path) id_funcs = {t.task.task_id:func for t, func in registry.rtc_runners.iteritems()} func = id_funcs.get(rtc.task.task_id, None) if func is None: sys.stderr.write("ERROR. Unknown tool contract id {x}".format(x=rtc.task.task_id)) return -1 else: log.info("Running id:{i} Resolved Tool Contract {r}".format(r=rtc, i=rtc.task.task_id)) log.info("Runner func {f}".format(f=func)) exit_code = func(rtc) log.info("Completed running {r} exitcode {e}".format(r=rtc, e=exit_code)) return exit_code
def _w(args): started_at = time.time() def run_time(): return time.time() - started_at def exit_msg(rcode_): return "Completed running {r} exitcode {e} in {t:.2f} sec.".format( r=rtc, e=rcode_, t=run_time()) level = get_parsed_args_log_level(args) setup_logger(None, level=level) log.info("Loading pbcommand {v}".format(v=pbcommand.get_version())) log.info("Registry {r}".format(r=registry)) log.info("Setting log-level to {d}".format(d=level)) log.debug("args {a}".format(a=args)) log.info("loading RTC from {i}".format(i=args.rtc_path)) rtc = load_resolved_tool_contract_from(args.rtc_path) id_funcs = { t.task.task_id: func for t, func in registry.rtc_runners.iteritems() } func = id_funcs.get(rtc.task.task_id, None) if func is None: rcode = 1 log.error( "Unknown tool contract id '{x}' Registered TC ids {i}".format( x=rtc.task.task_id, i=id_funcs.keys())) log.error(exit_msg(rcode)) return rcode else: log.info("Running id:{i} Resolved Tool Contract {r}".format( r=rtc, i=rtc.task.task_id)) log.info("Runner func {f}".format(f=func)) exit_code = func(rtc) if exit_code == 0: log.info(exit_msg(exit_code)) else: log.error(exit_msg(exit_code)) return exit_code
def _w(args): started_at = time.time() def run_time(): return time.time() - started_at def exit_msg(rcode_): return "Completed running {r} exitcode {e} in {t:.2f} sec.".format(r=rtc, e=rcode_, t=run_time()) level = get_parsed_args_log_level(args) setup_logger(None, level=level) log.info("Loading pbcommand {v}".format(v=pbcommand.get_version())) log.info("Registry {r}".format(r=registry)) log.info("Setting log-level to {d}".format(d=level)) log.debug("args {a}".format(a=args)) log.info("loading RTC from {i}".format(i=args.rtc_path)) rtc = load_resolved_tool_contract_from(args.rtc_path) id_funcs = {t.task.task_id: func for t, func in registry.rtc_runners.iteritems()} func = id_funcs.get(rtc.task.task_id, None) if func is None: rcode = 1 log.error("Unknown tool contract id '{x}' Registered TC ids {i}".format(x=rtc.task.task_id, i=id_funcs.keys())) log.error(exit_msg(rcode)) return rcode else: log.info("Running id:{i} Resolved Tool Contract {r}".format(r=rtc, i=rtc.task.task_id)) log.info("Runner func {f}".format(f=func)) exit_code = func(rtc) if exit_code == 0: log.info(exit_msg(exit_code)) else: log.error(exit_msg(exit_code)) return exit_code
def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs): """ Runs a general func and logs results. The return type is expected to be an (int) return code. :param alog: a log instance :param func: a cli exe func, must return an int exit code. func(args) => Int, where args is parsed from p.parse_args() :param args: parsed args from parser :param setup_log_func: F(alog, level=value, file_name=value, formatter=value) or None :return: Exit code of callable func :rtype: int """ started_at = time.time() pargs = args[0] # default logging level level = logging.INFO if 'level' in kwargs: level = kwargs.pop('level') else: level = get_parsed_args_log_level(pargs) # None will default to stdout log_file = getattr(pargs, 'log_file', None) # Currently, only support to stdout. More customization would require # more required commandline options in base parser (e.g., --log-file, # --log-formatter) log_options = dict(level=level, file_name=log_file) base_dir = os.getcwd() dump_alarm_on_error = False if "dump_alarm_on_error" in kwargs: dump_alarm_on_error = kwargs.pop("dump_alarm_on_error") is_cromwell_environment = bool( os.environ.get("SMRT_PIPELINE_BUNDLE_DIR", None)) and "cromwell-executions" in base_dir dump_alarm_on_error = dump_alarm_on_error and is_cromwell_environment # The Setup log func must adhere to the pbcommand.utils.setup_log func # signature # FIXME. This should use the more concrete F(file_name_or_name, level, formatter) # signature of setup_logger if setup_log_func is not None and alog is not None: setup_log_func(alog, **log_options) alog.info("Using pbcommand v{v}".format(v=pbcommand.get_version())) alog.info( "completed setting up logger with {f}".format(f=setup_log_func)) alog.info("log opts {d}".format(d=log_options)) if dump_alarm_on_error: alog.info( "This command appears to be running as part of a Cromwell workflow" ) alog.info("Additional output files may be generated") try: # the code in func should catch any exceptions. The try/catch # here is a fail safe to make sure the program doesn't fail # and the makes sure the exit code is logged. return_code = exe_main_func(*args, **kwargs) run_time = time.time() - started_at except Exception as e: run_time = time.time() - started_at if alog is not None: alog.error(e, exc_info=True) else: traceback.print_exc(sys.stderr) if dump_alarm_on_error: PacBioAlarm.dump_error(file_name=os.path.join( base_dir, "alarms.json"), exception=e, info="".join(traceback.format_exc()), message=str(e), name=e.__class__.__name__, severity=logging.ERROR) # We should have a standard map of exit codes to Int if isinstance(e, IOError): return_code = 1 else: return_code = 2 _d = dict(r=return_code, s=run_time) if is_cromwell_environment: alog.info("Writing task report to task-report.json") write_task_report(run_time, getattr(pargs, "nproc", 1), return_code) if alog is not None: alog.info("exiting with return code {r} in {s:.2f} sec.".format(**_d)) return return_code
#source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'pbcommand' copyright = u'2015-2016, Michael Kocher' author = u'Michael Kocher' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = pbcommand.get_version() # The full version, including alpha/beta/rc tags. release = pbcommand.get_version() # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y'
"""Common models for Tool Contract and Resolved Tool Contract Author: Michael Kocher """ import abc import pbcommand from .common import TaskTypes, ResourceTypes __version__ = pbcommand.get_version() class MalformedToolContractError(ValueError): pass class MalformedResolvedToolContractError(ValueError): pass def _validate_type(value, type_or_types): return isinstance(value, type_or_types) def _validate_or_raise(value, type_or_types): if not _validate_type(value, type_or_types): _d = dict(x=value, t=type(value), s=type_or_types) raise TypeError("Unsupported type for {x} {t}. Expected types {s}".format(**_d)) return value
"""Common models for Tool Contract and Resolved Tool Contract Author: Michael Kocher """ import abc from collections import OrderedDict import types import datetime import pbcommand from .common import TaskTypes, ResourceTypes, REGISTERED_FILE_TYPES __version__ = pbcommand.get_version() class MalformedToolContractError(ValueError): pass class MalformedResolvedToolContractError(ValueError): pass def _validate_type(value, type_or_types): return isinstance(value, type_or_types) def _validate_or_raise(value, type_or_types): if not _validate_type(value, type_or_types):
#source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'pbcommand' copyright = u'2015-2017, Michael Kocher' author = u'Michael Kocher' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = pbcommand.get_version() # The full version, including alpha/beta/rc tags. release = pbcommand.get_version() # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y'
def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs): """ Runs a general func and logs results. The return type is expected to be an (int) return code. :param alog: a log instance :param func: a cli exe func, must return an int exit code. func(args) => Int, where args is parsed from p.parse_args() :param args: parsed args from parser :param setup_log_func: F(alog, level=value, file_name=value, formatter=value) or None :return: Exit code of callable func :rtype: int """ started_at = time.time() pargs = args[0] # default logging level level = logging.INFO if 'level' in kwargs: level = kwargs.pop('level') else: level = get_parsed_args_log_level(pargs) # None will default to stdout log_file = getattr(pargs, 'log_file', None) # Currently, only support to stdout. More customization would require # more required commandline options in base parser (e.g., --log-file, --log-formatter) log_options = dict(level=level, file_name=log_file) # The Setup log func must adhere to the pbcommand.utils.setup_log func # signature # FIXME. This should use the more concrete F(file_name_or_name, level, formatter) # signature of setup_logger if setup_log_func is not None and alog is not None: setup_log_func(alog, **log_options) alog.info("Using pbcommand v{v}".format(v=pbcommand.get_version())) alog.info( "completed setting up logger with {f}".format(f=setup_log_func)) alog.info("log opts {d}".format(d=log_options)) try: # the code in func should catch any exceptions. The try/catch # here is a fail safe to make sure the program doesn't fail # and the makes sure the exit code is logged. return_code = exe_main_func(*args, **kwargs) run_time = time.time() - started_at except Exception as e: run_time = time.time() - started_at if alog is not None: alog.error(e, exc_info=True) else: traceback.print_exc(sys.stderr) # We should have a standard map of exit codes to Int if isinstance(e, IOError): return_code = 1 else: return_code = 2 _d = dict(r=return_code, s=run_time) if alog is not None: alog.info("exiting with return code {r} in {s:.2f} sec.".format(**_d)) return return_code
def _pacbio_main_runner(alog, setup_log_func, exe_main_func, *args, **kwargs): """ Runs a general func and logs results. The return type is expected to be an (int) return code. :param alog: a log instance :param func: a cli exe func, must return an int exit code. func(args) => Int, where args is parsed from p.parse_args() :param args: parsed args from parser :param setup_log_func: F(alog, level=value, file_name=value, formatter=value) or None :return: Exit code of callable func :rtype: int """ started_at = time.time() pargs = args[0] # default logging level level = logging.INFO if 'level' in kwargs: level = kwargs['level'] # If level is provided at the parser level, this will override all other # settings. This is defined in pbcommand.common_options.add_log_level_option if hasattr(pargs, 'log_level'): level = logging.getLevelName(pargs.log_level) # None will default to stdout log_file = getattr(pargs, 'log_file', None) # Currently, only support to stdout. More customization would require # more required commandline options in base parser (e.g., --log-file, --log-formatter) log_options = dict(level=level, file_name=log_file) # The Setup log func must adhere to the pbcommand.utils.setup_log func # signature # FIXME. This should use the more concrete F(file_name_or_name, level, formatter) # signature of setup_logger if setup_log_func is not None and alog is not None: setup_log_func(alog, **log_options) alog.info("Using pbcommand v{v}".format(v=pbcommand.get_version())) alog.info("completed setting up logger with {f}".format(f=setup_log_func)) alog.info("log opts {d}".format(d=log_options)) try: # the code in func should catch any exceptions. The try/catch # here is a fail safe to make sure the program doesn't fail # and the makes sure the exit code is logged. return_code = exe_main_func(*args, **kwargs) run_time = time.time() - started_at except Exception as e: run_time = time.time() - started_at if alog is not None: alog.error(e, exc_info=True) traceback.print_exc(sys.stderr) # We should have a standard map of exit codes to Int if isinstance(e, IOError): return_code = 1 else: return_code = 2 _d = dict(r=return_code, s=run_time) if alog is not None: alog.info("exiting with return code {r} in {s:.2f} sec.".format(**_d)) return return_code