def main(): parser = arg_parser() parsed_args = parser.parse_args(sys.argv[1:]) # Load the requested parsl configuration if parsed_args.parsl == 'cori': parsl.load(cori_regular_config) elif parsed_args.parsl == 'cori-debug': parsl.load(cori_debug_config) else: parsl.load(threads_config) # Trigger the argparse message if the cwl file is missing # Otherwise cwltool will use the default argparser if not parsed_args.workflow: if os.path.isfile("CWLFile"): setattr(parsed_args, "workflow", "CWLFile") else: _logger.error("") _logger.error("CWL document required, no input file was provided") parser.print_help() sys.exit(1) elif not parsed_args.basedir: _logger.error("") _logger.error("Basedir is required for storing itermediate results") parser.print_help() sys.exit(1) rc = RuntimeContext(vars(parsed_args)) rc.shifter = False parsed_args.__dict__['parallel'] = True rc.tmpdir_prefix = rc.basedir + '/tmp/tmp' rc.tmp_outdir_prefix = rc.basedir + '/out/out' # type: Text if parsed_args.shifter: rc.shifter = True rc.docker_outdir = '/spooldir' rc.docker_stagedir = rc.basedir + '/stage' rc.docker_tmpdir = '/tmpdir' lc = LoadingContext(vars(parsed_args)) lc.construct_tool_object = customMakeTool sys.exit( cwltool.main.main(args=parsed_args, loadingContext=lc, runtimeContext=rc))
def run_native(config_object: 'ConfigBase', workflow: str, run_directory: str = '.', verbosity="normal") -> int: """Executes the workflow using native Python rather than subprocess "command line" Args: config_object: a constructed ConfigBase-derived object workflow: the path to the workflow to be executed run_directory: the destination folder for workflow output subdirectories (default: CWD) parallel: process libraries in parallel where possible verbosity: controls the depth of information written to terminal by cwltool Returns: None """ def furnish_if_file_record(file_dict): if isinstance(file_dict, dict) and file_dict.get('class', None) == 'File': file_dict['basename'] = os.path.basename(file_dict['path']) file_dict['location'] = file_dict['path'] file_dict['contents'] = None # Upgrade file entries in Run Config with extra descriptors cwltool expects for _, config_param in config_object.config.items(): if isinstance(config_param, list): for config_dict in config_param: furnish_if_file_record(config_dict) else: furnish_if_file_record(config_param) # Set overall config for cwltool runtime_context = RuntimeContext({ 'secret_store': cwltool.secrets.SecretStore(), 'outdir': run_directory, 'on_error': "continue", 'js_console': verbosity == "debug", 'debug': verbosity == "debug" }) # Set proper temp directory for Mac users if sys.platform == "darwin": default_mac_path = "/private/tmp/docker_tmp" if runtime_context.tmp_outdir_prefix == DEFAULT_TMP_PREFIX: runtime_context.tmp_outdir_prefix = default_mac_path if runtime_context.tmpdir_prefix == DEFAULT_TMP_PREFIX: runtime_context.tmpdir_prefix = default_mac_path # Enable rich terminal output (timestamp, color, formatting) logger = logging.getLogger("cwltool") logger.handlers.clear() # executors.py loads a default handler; outputs are printed twice if we don't clear it level = 'DEBUG' if verbosity == 'debug' else 'WARN' if verbosity == "quiet" else "INFO" coloredlogs.install(logger=logger, stream=sys.stderr, fmt="[%(asctime)s] %(levelname)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=level, isatty=True) # Create a wrapper for the executors so that we may pass our logger to them (unsupported by Factory) parallel: MultithreadedJobExecutor = functools.partial(MultithreadedJobExecutor(), logger=logger) serial: SingleJobExecutor = functools.partial(SingleJobExecutor(), logger=logger) # Instantiate Factory with our run preferences cwl = cwltool.factory.Factory( runtime_context=runtime_context, loading_context=LoadingContext({'relax_path_checks': True}), executor=parallel if parallel else serial ) try: # Load the workflow document and execute pipeline = cwl.make(workflow) pipeline(**config_object.config) except cwltool.factory.WorkflowStatus: # For now, return non-zero if workflow did not complete return 1 return 0