def __init__(self, logger, path, args_files, args_data, req_files=[], req_keys=[]): self.log = logger self.conf_path = path self.conf_keys = req_keys self.required_conf_files = [os.path.join(self.conf_path, cf) for cf in req_files] self.conf_files = [] if args_files is not None: for cf in args_files: if not os.path.isabs(cf): cf = os.path.join(os.getcwd(), cf) self.conf_files += [cf] self.conf_data = args_data or [] self.conf_builder = ConfigBuilder()
class RunCommand(Command):
DEFAULT_CONF_FILES = [
"system.conf"
]
DEFAULT_REQUIRED_CONF = [
# "work_path", "temp_path"]
]
def __init__(self, args_usage="", epilog="", flow_file=None, conf_files=None, conf_keys=None, logger_name=None):
Command.__init__(self, args_usage, epilog, logger_name)
signal.signal(signal.SIGINT, keyboardinterrupt_handler)
signal.signal(signal.SIGTERM, keyboardinterrupt_handler)
if conf_files is None:
conf_files = []
if conf_keys is None:
conf_keys = []
self.flow_file = flow_file
self.conf_files = self.DEFAULT_CONF_FILES + conf_files
self.conf_keys = self.DEFAULT_REQUIRED_CONF + conf_keys
# Workspace
self.workspace = self.args.workspace
# Instance name
self.instance_name = self.args.instance_name
if self.instance_name is not None:
self.instance_name = normalize_id(self.instance_name)
'''
# Override configuration path if required
if self.args.conf_path is not None:
self.conf_path = os.path.abspath(self.args.conf_path)
# Get required configuration files and override system.conf if required
if self.args.system_conf is not None:
req_conf_files = []
for cf in self.conf_files:
if cf == "system.conf":
req_conf_files += [self.args.system_conf]
else:
req_conf_files += [cf]
else:
req_conf_files = self.conf_files
'''
req_conf_files = self.conf_files
# Determine required and user configuration files and data
self.required_conf_files = [os.path.join(self.conf_path, cf)
for cf in req_conf_files]
if self.args.conf_files is not None:
self.user_conf_files = []
for cf in self.args.conf_files:
if not os.path.isabs(cf):
cf = os.path.join(os.getcwd(), cf)
self.user_conf_files += [cf]
else:
self.user_conf_files = []
if self.args.conf_data is not None:
self.user_conf_data = self.args.conf_data
else:
self.user_conf_data = []
# max cores
if self.args.max_cores is None:
self.max_cores = 0
else:
self.max_cores = self.args.max_cores
# Prepare extra configuration data
self.extra_conf_data = []
def add_arguments(self, parser):
"Called from __init__() to let the command to add more options to the OptionsParser"
parser.add_argument("-w", "--workspace", dest = "workspace", metavar = "NAME", default="default",
help = "Define the workspace name.")
parser.add_argument("-u", "--user", dest="user", metavar="NAME",
help="Define the user name.")
g = parser.add_argument_group("Wok Options")
g.add_argument("-n", "--instance", dest = "instance_name", metavar = "NAME",
help = "Define the instance name")
g.add_argument("-C", "--conf", action="append", dest="conf_files", metavar="FILE",
help="Load configuration from a file. Multiple files can be specified")
g.add_argument("-D", action="append", dest="conf_data", metavar="PARAM=VALUE",
help="Define configuration parameter. Multiple definitions can be specified. Example: -D option1=value")
g.add_argument("-j", dest="max_cores", metavar="NUM", type=int,
help="Define the maximum number of cores to use. Default all the cores available.")
def build_conf(self):
"Called from run() to prepare configuration before expansion of values"
conf_files = self.required_conf_files
user_conf_file = os.path.join(self.conf_path, "user.conf")
if os.path.exists(user_conf_file):
conf_files += [user_conf_file]
conf_files += self.user_conf_files
# Check that configuration files exist
missing_conf_files = [cf for cf in conf_files if not os.path.exists(cf)]
if len(missing_conf_files) > 0:
self.log.error("Configuration files not found:\n{}".format(
"\n".join(" {}".format(cf) for cf in missing_conf_files)))
exit(-1)
# Build the configuration
self.conf_builder = ConfigBuilder()
if self.instance_name is None:
if self.workspace is not None:
self.instance_name = self.workspace
else:
self.instance_name = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f")
self.conf_builder.add_value("wok.instance.name", self.instance_name)
# Files
for cf in conf_files:
self.conf_builder.add_file(cf)
# General conf
self.conf_builder.add_value("workspace", self.workspace)
self.conf_builder.add_value("workflows_path", self.workflows_path)
if self.max_cores > 0:
self.conf_builder.add_value("wok.platform.jobs.max_cores", self.max_cores)
if self.args.log_level is not None:
self.conf_builder.add_value("log.level", self.args.log_level)
self.conf_builder.add_value("wok.engine", self.args.log_level)
# Incorporate user and extra configuration data
for data in self.extra_conf_data + self.user_conf_data:
try:
pos = data.index("=")
key = data[0:pos]
value = data[pos+1:]
try:
v = json.loads(value)
except:
v = value
self.conf_builder.add_value(key, Data.create(v))
except:
raise Exception("Wrong configuration data: KEY=VALUE expected but found '{}'".format(data))
def process_conf(self):
"Called from run() to process the configuration after expansion of values"
# Validate that required keys exist
mk = self.conf.missing_fields(self.conf_keys)
if len(mk) > 0:
sb = ["The following configuration parameters were not found:\n",
"\n".join("* " + k for k in mk),
"\nin any of the following configuration files:\n",
"\n".join("* " + k for k in conf_files)]
self.log.error("".join(sb))
exit(-1)
# Configurable paths
if "runtime_path" in self.conf:
self.runtime_path = self.conf["runtime_path"]
else:
self.conf_builder.add_value("runtime_path", self.runtime_path)
# Configure wok work path
if "wok.work_path" not in self.conf:
self.conf_builder.add_value("wok.work_path", os.path.join(self.runtime_path, "wok"))
# Prepare paths by user_id
user_id = self.args.user or getpass.getuser()
rpath = self.conf.get("results_path", os.path.join(self.runtime_path, "results"))
self.results_path = os.path.join(rpath, user_id)
self.conf_builder.add_value("results_path", self.results_path)
if not os.path.exists(self.results_path):
os.makedirs(self.results_path)
tpath = self.conf.get("temp_path", os.path.join(self.runtime_path, "temp"))
self.temp_path = os.path.join(tpath, user_id)
self.conf_builder.add_value("temp_path", self.temp_path)
if not os.path.exists(self.temp_path):
os.makedirs(self.temp_path)
def run(self):
"Run the command and execute the command"
self.build_conf()
# Expand configuration variables
self.conf = self.conf_builder.get_conf()
self.conf.expand_vars()
#self.log.debug(repr(self.conf))
# Validate and process configuration
self.process_conf()
# Regenerate configuration
self.conf = self.conf_builder.get_conf()
# Final logging configuration
log = logger.get_logger("")
log.removeHandler(log.handlers[0])
logging_conf = self.conf.get("wok.logging")
logger.initialize(logging_conf)
# Show some debugging information
self.log.debug("Root path = {}".format(self.root_path))
self.log.debug("Conf path = {}".format(self.conf_path))
self.log.debug("Data path = {}".format(self.data_path))
self.log.debug("Workflows path = {}".format(self.workflows_path))
self.log.debug("Runtime path = {}".format(self.runtime_path))
self.log.debug("Results path = {}".format(self.results_path))
self.log.debug("Temp path = {}".format(self.temp_path))
if len(self.user_conf_files) > 0:
self.log.debug("User defined configuration files:\n{}".format(
"\n".join(" {}".format(cf) for cf in self.user_conf_files)))
self.log.debug("Effective configuration: " + str(self.conf))
# Execute
try:
self.execute()
except Exception as ex:
self.log.exception(ex)
return -1
return 0
def _wok_run(self, case_name, conf_builder, flow_uri):
wok = WokEngine(self.conf)
wok.start(wait=False, single_run=True)
try:
case = wok.create_case(case_name, conf_builder, flow_uri)
wok.wait()
except KeyboardInterrupt:
pass
finally:
try:
wok.stop()
except KeyboardInterrupt:
self.log.warn("Ctrl-C pressed while stopping Wok engine.")
class ConfArgs(object):
def __init__(self, logger, path, args_files, args_data, req_files=[], req_keys=[]):
self.log = logger
self.conf_path = path
self.conf_keys = req_keys
self.required_conf_files = [os.path.join(self.conf_path, cf) for cf in req_files]
self.conf_files = []
if args_files is not None:
for cf in args_files:
if not os.path.isabs(cf):
cf = os.path.join(os.getcwd(), cf)
self.conf_files += [cf]
self.conf_data = args_data or []
self.conf_builder = ConfigBuilder()
def load_files(self):
"Called from run() to prepare configuration before expansion of values"
conf_files = self.required_conf_files
user_conf_file = os.path.join(self.conf_path, "user.conf")
if os.path.exists(user_conf_file):
conf_files += [user_conf_file]
conf_files += self.conf_files
# Check that configuration files exist
missing_conf_files = [cf for cf in conf_files if not os.path.exists(cf)]
if len(missing_conf_files) > 0:
self.log.error("Configuration files not found:\n{}".format(
"\n".join(" {}".format(cf) for cf in missing_conf_files)))
exit(-1)
for cf in conf_files:
self.conf_builder.add_file(cf)
return self.conf_builder
def parse_data(self):
for data in self.conf_data:
try:
pos = data.index("=")
key = data[0:pos]
value = data[pos+1:]
try:
v = json.loads(value)
except:
v = value
self.conf_builder.add_value(key, v)
except:
raise Exception("Wrong configuration data: KEY=VALUE expected but found '{}'".format(data))
return self.conf_builder
def validated_conf(self, expand_vars=True):
self.conf = self.conf_builder.get_conf()
if expand_vars:
self.conf.expand_vars()
#self.log.debug(repr(self.conf))
# Validate that required keys exist
mk = self.conf.missing_keys(self.conf_keys)
if len(mk) > 0:
sb = ["The following configuration parameters were not found:\n",
"\n".join("* " + k for k in mk),
"\nin any of the following configuration files:\n",
"\n".join("* " + k for k in conf_files)]
self.log.error("".join(sb))
exit(-1)
return self.conf
def log_debug(self):
if len(self.conf_files) > 0:
self.log.debug("User defined configuration files:\n{}".format(
"\n".join(" {}".format(cf) for cf in self.conf_files)))
self.log.debug("Effective configuration: " + str(self.conf))
def build_conf(self):
"Called from run() to prepare configuration before expansion of values"
conf_files = self.required_conf_files
user_conf_file = os.path.join(self.conf_path, "user.conf")
if os.path.exists(user_conf_file):
conf_files += [user_conf_file]
conf_files += self.user_conf_files
# Check that configuration files exist
missing_conf_files = [cf for cf in conf_files if not os.path.exists(cf)]
if len(missing_conf_files) > 0:
self.log.error("Configuration files not found:\n{}".format(
"\n".join(" {}".format(cf) for cf in missing_conf_files)))
exit(-1)
# Build the configuration
self.conf_builder = ConfigBuilder()
if self.instance_name is None:
if self.workspace is not None:
self.instance_name = self.workspace
else:
self.instance_name = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f")
self.conf_builder.add_value("wok.instance.name", self.instance_name)
# Files
for cf in conf_files:
self.conf_builder.add_file(cf)
# General conf
self.conf_builder.add_value("workspace", self.workspace)
self.conf_builder.add_value("workflows_path", self.workflows_path)
if self.max_cores > 0:
self.conf_builder.add_value("wok.platform.jobs.max_cores", self.max_cores)
if self.args.log_level is not None:
self.conf_builder.add_value("log.level", self.args.log_level)
self.conf_builder.add_value("wok.engine", self.args.log_level)
# Incorporate user and extra configuration data
for data in self.extra_conf_data + self.user_conf_data:
try:
pos = data.index("=")
key = data[0:pos]
value = data[pos+1:]
try:
v = json.loads(value)
except:
v = value
self.conf_builder.add_value(key, Data.create(v))
except:
raise Exception("Wrong configuration data: KEY=VALUE expected but found '{}'".format(data))
