def _wok_run(self, case_name, conf_builder, flow_uri):
wok = WokEngine(self.conf)
wok.start(wait=False, single_run=True)
try:
case = wok.create_case(case_name, conf_builder, flow_uri)
wok.wait()
except KeyboardInterrupt:
pass
finally:
try:
wok.stop()
except KeyboardInterrupt:
self.log.warn("Ctrl-C pressed while stopping Wok engine.")
class RunCommand(Command):
DEFAULT_CONF_FILES = [
"system.conf", "run.conf"
]
DEFAULT_REQUIRED_CONF = [
# "work_path", "temp_path"]
]
def __init__(self, args_usage="", epilog="", logger_name=None):
Command.__init__(self, args_usage, epilog, logger_name)
signal.signal(signal.SIGINT, keyboardinterrupt_handler)
signal.signal(signal.SIGTERM, keyboardinterrupt_handler)
'''
# Override configuration path if required
if self.args.conf_path is not None:
self.conf_path = os.path.abspath(self.args.conf_path)
'''
# Determine required and user configuration files and data
self.engine_conf_args = ConfArgs(self.log, self.conf_path,
self.args.engine_conf_files, self.args.engine_conf_data,
self.DEFAULT_CONF_FILES, self.DEFAULT_REQUIRED_CONF)
self.engine_conf_builder = self.engine_conf_args.conf_builder
self.case_conf_args = ConfArgs(self.log, self.conf_path, self.args.case_conf_files, self.args.case_conf_data)
self.case_conf_builder = self.case_conf_args.conf_builder
# Workspace
self.workspace = self.args.workspace
# Case name
self.case_name = self.args.case_name
if self.case_name is not None:
self.case_name = normalize_id(self.case_name)
# max cores
if self.args.max_cores is None:
self.max_cores = 0
else:
self.max_cores = self.args.max_cores
def add_arguments(self, parser):
"Called from __init__() to let the command to add more options to the OptionsParser"
parser.add_argument("-w", "--workspace", dest = "workspace", metavar = "NAME", default="default",
help = "Define the workspace name.")
parser.add_argument("-u", "--user", dest="user", metavar="NAME",
help="Define the user name.")
parser.add_argument("-s", "--storage", dest="storage", metavar="NAME",
help="Define the storage name.")
g = parser.add_argument_group("Wok Options")
g.add_argument("-n", "--case", dest = "case_name", metavar = "NAME",
help = "Define the case name")
g.add_argument("-c", "--engine-conf", action="append", dest="engine_conf_files", metavar="FILE",
help="Load engine configuration from a file. Multiple files can be specified")
g.add_argument("-d", action="append", dest="engine_conf_data", metavar="PARAM=VALUE",
help="Define engine configuration parameter. Multiple definitions can be specified. Example: -d option1=value")
g.add_argument("-C", "--conf", action="append", dest="case_conf_files", metavar="FILE",
help="Load case configuration from a file. Multiple files can be specified")
g.add_argument("-D", action="append", dest="case_conf_data", metavar="PARAM=VALUE",
help="Define case configuration parameter. Multiple definitions can be specified. Example: -D option1=value")
g.add_argument("-j", dest="max_cores", metavar="NUM", type=int,
help="Define the maximum number of cores to use. Default all the cores available.")
def build_conf(self):
"Called from run() to prepare configuration before expansion of values"
# Build the configuration
self.engine_conf_args.load_files()
self.case_conf_args.load_files()
# General conf
if self.case_name is None:
if self.workspace is not None:
self.case_name = self.workspace
else:
self.case_name = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f")
if self.max_cores > 0:
self.engine_conf_builder.add_value("wok.platforms[0].jobs.max_cores", self.max_cores)
if self.args.log_level is not None:
self.case_conf_builder.add_value("logging", self.args.log_level)
# Set user and workspace
self.user_id = self.args.user or getpass.getuser()
self.container = self.args.storage
self.case_conf_builder.add_value("user_id", self.user_id)
self.case_conf_builder.add_value("workspace", self.workspace)
# Parse conf data
self.engine_conf_args.parse_data()
self.case_conf_args.parse_data()
def process_conf(self):
"Called from run() to process the configuration after expansion of values"
# Configure wok work path
if "wok.work_path" not in self.engine_conf:
self.engine_conf_builder.add_value("wok.work_path",
os.path.join(self.engine_conf.get("runtime_path", os.getcwd()), "wok"))
def run(self):
"Run the command and execute the command"
self.build_conf()
# Expand configuration variables
self.engine_conf = self.engine_conf_args.validated_conf()
self.case_conf = self.case_conf_args.validated_conf(expand_vars=False)
# Validate and process configuration
self.process_conf()
# Regenerate configuration
self.engine_conf = self.engine_conf_builder.get_conf()
self.case_conf = self.case_conf_builder.get_conf()
# Final logging configuration
log = logger.get_logger("")
log.removeHandler(log.handlers[0])
logging_conf = self.engine_conf.get("wok.logging")
logger.initialize(logging_conf)
# Show some debugging information
self.log.debug("Root path = {}".format(self.root_path))
self.log.debug("Conf path = {}".format(self.conf_path))
#self.log.debug("Runtime path = {}".format(self.runtime_path))
#self.log.debug("Results path = {}".format(self.results_path))
#self.log.debug("Temp path = {}".format(self.temp_path))
self.engine_conf_args.log_debug()
self.case_conf_args.log_debug()
# Execute
try:
self.execute()
except BaseException as ex:
self.log.error(str(ex))
from traceback import format_exc
self.log.debug(format_exc())
return -1
return 0
def _case_created(self, case, **kwargs):
if hasattr(self, "projects") and self.projects is not None:
upload_files(self.log, case.name, case.storages, self.projects)
config = GlobalConfig(
case.project.get_conf(platform_name=DEFAULT_PLATFORM_NAME),
dict(user_id=self.user_id, workspace=self.workspace))
self.results_path = PathsConfig(config).results_path()
def _case_finished(self, case, **kwargs):
if hasattr(self, "results_path") and self.results_path is not None:
download_files(self.log, case.name, case.storages, "results/", self.results_path)
def _wok_run(self, flow, case_name=None, project=None, container=None):
# Create and start the Wok engine
self.wok = WokEngine(self.engine_conf, self.conf_path)
self.wok.case_created.connect(self._case_created)
self.wok.case_finished.connect(self._case_finished)
self.wok.start(wait=False, single_run=True)
# Create and start the case
try:
case_name = case_name or self.case_name
case = self.wok.create_case(
case_name,
self.case_conf_builder,
project or PROJECT_NAME,
flow,
container or self.container or case_name)
case.start()
self.wok.wait()
except KeyboardInterrupt:
pass
finally:
try:
self.wok.stop()
except KeyboardInterrupt:
self.log.warn("Ctrl-C pressed while stopping Wok engine.")
