def _get_process_url_schema():
return OP.to_option_schema(
_to_wopt_id("progress_status_url"),
("string", "null"),
"Status Progress URL",
"Post status progress updates to URL.",
None,
)
def _get_exit_on_failure():
return OP.to_option_schema(
_to_wopt_id("debug_mode"),
"boolean",
"Enable Debug Mode",
"Debug will emit debug messages to Stdout and set the level in the master log to DEBUG.",
GlobalConstants.DEBUG_MODE,
)
def _to_max_nproc_option():
return OP.to_option_schema(
_to_wopt_id("max_total_nproc"),
("integer", "null"),
"Maximum Total Number of Processors",
"Maximum Total number of Processors/Slots the workflow engine will use (null means there is no limit).",
GlobalConstants.MAX_TOTAL_NPROC,
)
def _to_max_nproc_option():
return OP.to_option_schema(
_to_wopt_id("max_nproc"),
"integer",
"Maximum Total Number of Processors Per Task",
"Maximum number of Processors per Task.",
GlobalConstants.MAX_NPROC,
)
def _get_workflow_option_schema():
return OP.to_option_schema(
_to_wopt_id("max_nworkers"),
"integer",
"Max Number of Workers",
"Max Number of concurrently running tasks. (Note: max_nproc will restrict the number of workers if max_nworkers * max_nproc > max_total_nproc)",
GlobalConstants.MAX_NWORKERS,
)
def _get_chunked_mode_schema():
return OP.to_option_schema(
_to_wopt_id("chunk_mode"),
"boolean",
"Chunked File Mode",
"Enable file splitting (chunking) mode",
GlobalConstants.CHUNKED_MODE,
)
def _get_distributed_mode_schema():
return OP.to_option_schema(
_to_wopt_id("distributed_mode"),
"boolean",
"Distributed File Mode",
"Enable Distributed mode to submit jobs to the cluster. (Must provide 'cluster_manager' path to cluster templates)",
GlobalConstants.DISTRIBUTED_MODE,
)
def _to_max_chunks_option():
return OP.to_option_schema(
_to_wopt_id("max_nchunks"),
"integer",
"Max Number of Chunks",
"Max Number of chunks that a file will be scattered into",
GlobalConstants.MAX_NCHUNKS,
)
def _get_exit_on_failure():
return OP.to_option_schema(
_to_wopt_id("exit_on_failure"),
"boolean",
"Exit On Failure",
"Immediately exit if a task fails (Instead of trying to run as many tasks as possible before exiting.)",
GlobalConstants.EXIT_ON_FAILIURE,
)
def _get_cluster_manager_schema():
return OP.to_option_schema(
_to_wopt_id("cluster_manager"),
("string", "null"),
"Cluster Template Path",
"Path to Cluster template files directory. The directory must contain 'start.tmpl', and 'stop.tmpl'"
"Or the path can be provided to a python module (e.g., 'pbsmrtpipe.cluster_templates.sge')",
"pbsmrtpipe.cluster_templates.sge_pacbio",
)
def _get_node_tmp_dir_schema():
return OP.to_option_schema(
_to_wopt_id("tmp_dir"),
("string", "null"),
"Temp directory",
"Temporary directory (/tmp) on the execution node. If running in distributed mode, "
"the tmp directory must be on the head node too.",
GlobalConstants.TMP_DIR,
)
def _get_simple_opts():
# Util func to create an task option id 'pbsmrtpipe.task_options.dev.hello_message'
oid = OP.to_opt_id('dev.hello_message')
return {oid: OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message")}
def _get_exit_on_failure():
return OP.to_option_schema(_to_wopt_id("debug_mode"), "boolean", "Enable Debug Mode",
"Debug will emit debug messages to Stdout and set the level in the master log to DEBUG.", False)
import pbsmrtpipe.schema_opt_utils as OP
# generate a task option id
oid = OP.to_opt_id('dev.hello_message')
# generate a schema
s = OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message")
print {oid: s}
def _get_workflow_option_schema():
return OP.to_option_schema(
_to_wopt_id("max_nworkers"), "integer", "Max Number of Workers",
"Max Number of concurrently running tasks. (Note: max_nproc will restrict the number of workers if max_nworkers * max_nproc > max_total_nproc)",
GlobalConstants.MAX_NWORKERS)
def _get_chunked_mode_schema():
return OP.to_option_schema(_to_wopt_id("chunk_mode"), "boolean",
"Chunked File Mode",
"Enable file splitting (chunking) mode",
GlobalConstants.CHUNKED_MODE)
def _get_distributed_mode_schema():
return OP.to_option_schema(
_to_wopt_id("distributed_mode"), "boolean", "Distributed File Mode",
"Enable Distributed mode to submit jobs to the cluster. (Must provide 'cluster_manager' path to cluster templates)",
GlobalConstants.DISTRIBUTED_MODE)
def _get_node_tmp_dir_schema():
return OP.to_option_schema(
_to_wopt_id("tmp_dir"), ("string", "null"), "Temp directory",
"Temporary directory (/tmp) on the execution node. If running in distributed mode, "
"the tmp directory must be on the head node too.",
GlobalConstants.TMP_DIR)
def _to_max_nproc_option():
return OP.to_option_schema(
_to_wopt_id("max_total_nproc"), ("integer", "null"),
"Maximum Total Number of Processors",
"Maximum Total number of Processors/Slots the workflow engine will use (null means there is no limit).",
GlobalConstants.MAX_TOTAL_NPROC)
def _get_chunked_mode_schema():
return OP.to_option_schema(_to_wopt_id("chunk_mode"), "boolean",
"Chunked File Mode",
"Enable file splitting (chunking) mode", False)
def _get_exit_on_failure():
return OP.to_option_schema(_to_wopt_id("exit_on_failure"), "boolean", "Exit On Failure",
"Immediately exit if a task fails (Instead of trying to run as many tasks as possible before exiting.", False)
def _get_exit_on_failure():
return OP.to_option_schema(
_to_wopt_id("exit_on_failure"), "boolean", "Exit On Failure",
"Immediately exit if a task fails (Instead of trying to run as many tasks as possible before exiting.)",
GlobalConstants.EXIT_ON_FAILIURE)
def _get_process_url_schema():
return OP.to_option_schema(_to_wopt_id("progress_status_url"),
("string", "null"), "Status Progress URL",
"Post status progress updates to URL.", None)
def _get_debug_model_option():
return OP.to_option_schema(
_to_wopt_id("debug_mode"), "boolean", "Enable Debug Mode",
"Debug will emit debug messages to Stdout and set the level in the master log to DEBUG.",
GlobalConstants.DEBUG_MODE)
def _get_cluster_manager_schema():
return OP.to_option_schema(_to_wopt_id("cluster_manager"), ("string", "null"),
"Cluster Template Path",
"Path to Cluster template files directory. The directory must contain 'start.tmpl', 'interactive.tmpl' and 'kill.tmpl' "
"Or path to python module (e.g., 'pbsmrtpipe.cluster_templates.sge')", "pbsmrtpipe.cluster_templates.sge_pacbio")
def _to_max_chunks_option():
return OP.to_option_schema(
_to_wopt_id("max_nchunks"), "integer", "Max Number of Chunks",
"Max Number of chunks that a file will be scattered into",
GlobalConstants.MAX_NCHUNKS)
def _to_max_nproc_option():
return OP.to_option_schema(_to_wopt_id("max_nproc"), "integer",
"Maximum Total Number of Processors Per Task",
"Maximum number of Processors per Task.",
GlobalConstants.MAX_NPROC)
def _get_opts():
oid = OP.to_opt_id('dev.hello_message')
return {oid: OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message")}
def _to_max_chunks_option():
return OP.to_option_schema(_to_wopt_id("max_nchunks"), "integer", "Max Number of Chunks",
"Max Number of chunks that a file will be scattered into", 10)
