def _wok_run(self, case_name, conf_builder, flow_uri): wok = WokEngine(self.conf) wok.start(wait=False, single_run=True) try: case = wok.create_case(case_name, conf_builder, flow_uri) wok.wait() except KeyboardInterrupt: pass finally: try: wok.stop() except KeyboardInterrupt: self.log.warn("Ctrl-C pressed while stopping Wok engine.")
class RunCommand(Command): DEFAULT_CONF_FILES = [ "system.conf", "run.conf" ] DEFAULT_REQUIRED_CONF = [ # "work_path", "temp_path"] ] def __init__(self, args_usage="", epilog="", logger_name=None): Command.__init__(self, args_usage, epilog, logger_name) signal.signal(signal.SIGINT, keyboardinterrupt_handler) signal.signal(signal.SIGTERM, keyboardinterrupt_handler) ''' # Override configuration path if required if self.args.conf_path is not None: self.conf_path = os.path.abspath(self.args.conf_path) ''' # Determine required and user configuration files and data self.engine_conf_args = ConfArgs(self.log, self.conf_path, self.args.engine_conf_files, self.args.engine_conf_data, self.DEFAULT_CONF_FILES, self.DEFAULT_REQUIRED_CONF) self.engine_conf_builder = self.engine_conf_args.conf_builder self.case_conf_args = ConfArgs(self.log, self.conf_path, self.args.case_conf_files, self.args.case_conf_data) self.case_conf_builder = self.case_conf_args.conf_builder # Workspace self.workspace = self.args.workspace # Case name self.case_name = self.args.case_name if self.case_name is not None: self.case_name = normalize_id(self.case_name) # max cores if self.args.max_cores is None: self.max_cores = 0 else: self.max_cores = self.args.max_cores def add_arguments(self, parser): "Called from __init__() to let the command to add more options to the OptionsParser" parser.add_argument("-w", "--workspace", dest = "workspace", metavar = "NAME", default="default", help = "Define the workspace name.") parser.add_argument("-u", "--user", dest="user", metavar="NAME", help="Define the user name.") parser.add_argument("-s", "--storage", dest="storage", metavar="NAME", help="Define the storage name.") g = parser.add_argument_group("Wok Options") g.add_argument("-n", "--case", dest = "case_name", metavar = "NAME", help = "Define the case name") g.add_argument("-c", "--engine-conf", action="append", dest="engine_conf_files", metavar="FILE", help="Load engine configuration from a file. Multiple files can be specified") g.add_argument("-d", action="append", dest="engine_conf_data", metavar="PARAM=VALUE", help="Define engine configuration parameter. Multiple definitions can be specified. Example: -d option1=value") g.add_argument("-C", "--conf", action="append", dest="case_conf_files", metavar="FILE", help="Load case configuration from a file. Multiple files can be specified") g.add_argument("-D", action="append", dest="case_conf_data", metavar="PARAM=VALUE", help="Define case configuration parameter. Multiple definitions can be specified. Example: -D option1=value") g.add_argument("-j", dest="max_cores", metavar="NUM", type=int, help="Define the maximum number of cores to use. Default all the cores available.") def build_conf(self): "Called from run() to prepare configuration before expansion of values" # Build the configuration self.engine_conf_args.load_files() self.case_conf_args.load_files() # General conf if self.case_name is None: if self.workspace is not None: self.case_name = self.workspace else: self.case_name = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f") if self.max_cores > 0: self.engine_conf_builder.add_value("wok.platforms[0].jobs.max_cores", self.max_cores) if self.args.log_level is not None: self.case_conf_builder.add_value("logging", self.args.log_level) # Set user and workspace self.user_id = self.args.user or getpass.getuser() self.container = self.args.storage self.case_conf_builder.add_value("user_id", self.user_id) self.case_conf_builder.add_value("workspace", self.workspace) # Parse conf data self.engine_conf_args.parse_data() self.case_conf_args.parse_data() def process_conf(self): "Called from run() to process the configuration after expansion of values" # Configure wok work path if "wok.work_path" not in self.engine_conf: self.engine_conf_builder.add_value("wok.work_path", os.path.join(self.engine_conf.get("runtime_path", os.getcwd()), "wok")) def run(self): "Run the command and execute the command" self.build_conf() # Expand configuration variables self.engine_conf = self.engine_conf_args.validated_conf() self.case_conf = self.case_conf_args.validated_conf(expand_vars=False) # Validate and process configuration self.process_conf() # Regenerate configuration self.engine_conf = self.engine_conf_builder.get_conf() self.case_conf = self.case_conf_builder.get_conf() # Final logging configuration log = logger.get_logger("") log.removeHandler(log.handlers[0]) logging_conf = self.engine_conf.get("wok.logging") logger.initialize(logging_conf) # Show some debugging information self.log.debug("Root path = {}".format(self.root_path)) self.log.debug("Conf path = {}".format(self.conf_path)) #self.log.debug("Runtime path = {}".format(self.runtime_path)) #self.log.debug("Results path = {}".format(self.results_path)) #self.log.debug("Temp path = {}".format(self.temp_path)) self.engine_conf_args.log_debug() self.case_conf_args.log_debug() # Execute try: self.execute() except BaseException as ex: self.log.error(str(ex)) from traceback import format_exc self.log.debug(format_exc()) return -1 return 0 def _case_created(self, case, **kwargs): if hasattr(self, "projects") and self.projects is not None: upload_files(self.log, case.name, case.storages, self.projects) config = GlobalConfig( case.project.get_conf(platform_name=DEFAULT_PLATFORM_NAME), dict(user_id=self.user_id, workspace=self.workspace)) self.results_path = PathsConfig(config).results_path() def _case_finished(self, case, **kwargs): if hasattr(self, "results_path") and self.results_path is not None: download_files(self.log, case.name, case.storages, "results/", self.results_path) def _wok_run(self, flow, case_name=None, project=None, container=None): # Create and start the Wok engine self.wok = WokEngine(self.engine_conf, self.conf_path) self.wok.case_created.connect(self._case_created) self.wok.case_finished.connect(self._case_finished) self.wok.start(wait=False, single_run=True) # Create and start the case try: case_name = case_name or self.case_name case = self.wok.create_case( case_name, self.case_conf_builder, project or PROJECT_NAME, flow, container or self.container or case_name) case.start() self.wok.wait() except KeyboardInterrupt: pass finally: try: self.wok.stop() except KeyboardInterrupt: self.log.warn("Ctrl-C pressed while stopping Wok engine.")