def main():
conf = OptionsConfig()
if "wok" not in conf:
print("Missing wok configuration")
exit(-1)
conf.expand_vars()
logger.initialize(conf.get("wok.logging"))
log = logger.get_logger("wok.server.start")
# run the server
retcode = 0
try:
server = WokServer(conf, app)
server.init()
server.run()
except Exception as e:
log.exception(e)
retcode = -1
exit(retcode)
def main():
conf = OptionsConfig()
if "wok" not in conf:
print("Missing wok configuration")
exit(-1)
conf.expand_vars()
logger.initialize(conf.get("wok.logging"))
log = logger.get_logger("wok.server.start")
# run the server
retcode = 0
try:
server = WokServer(conf, app)
server.init()
server.run()
except Exception as e:
log.exception(e)
retcode = -1
exit(retcode)
def __init__(self, args_usage="", epilog="", logger_name=None):
self.args_usage = args_usage
self.epilog = epilog
if logger_name is None:
if hasattr(self, "cmd_name"):
logger_name = self.cmd_name
else:
logger_name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
# Determine binary path
script_path = os.path.dirname(sys.argv[0])
if script_path:
self.launchers_path = os.path.abspath(script_path)
else:
self.launchers_path = os.getcwd()
# Determine default paths
self.root_path = os.path.normpath(os.path.join(self.launchers_path, ".."))
self.bin_path = os.path.join(self.root_path, "bin")
self.data_path = os.path.join(self.root_path, "data")
self.conf_path = os.path.join(self.root_path, "conf")
self.workflows_path = os.path.join(self.root_path, "workflows")
self.runtime_path = os.path.join(self.root_path, "runtime")
# Parse arguments
parser = ArgumentParser(
prog = "run " + self.cmd_name,
epilog = self.epilog,
formatter_class=RawDescriptionHelpFormatter)
parser.add_argument('--version', action='version', version='%(prog)s ' + VERSION)
self.add_arguments(parser)
parser.add_argument("-L", "--log-level", dest="log_level",
default=None, choices=["debug", "info", "warn", "error", "critical", "notset"],
help="Define the logging level")
self.args = parser.parse_args()
# Create preliminary logger
level = self.args.log_level or "info"
logger.initialize(level=level)
self.log = logger.get_logger(logger_name, level=level)
def main():
conf = Config()
initialize({"log":{"level":"debug"}})
log = get_logger("mrna_preproc_counts")
log.setLevel(logging.DEBUG)
log.info("Querying log2r tumour units ...")
es = EntityServer(conf["entities"])
em = es.manager()
def run(self):
"Run the command and execute the command"
self.build_conf()
# Expand configuration variables
self.conf = self.conf_builder.get_conf()
self.conf.expand_vars()
#self.log.debug(repr(self.conf))
# Validate and process configuration
self.process_conf()
# Regenerate configuration
self.conf = self.conf_builder.get_conf()
# Final logging configuration
log = logger.get_logger("")
log.removeHandler(log.handlers[0])
logging_conf = self.conf.get("wok.logging")
logger.initialize(logging_conf)
# Show some debugging information
self.log.debug("Root path = {}".format(self.root_path))
self.log.debug("Conf path = {}".format(self.conf_path))
self.log.debug("Data path = {}".format(self.data_path))
self.log.debug("Workflows path = {}".format(self.workflows_path))
self.log.debug("Runtime path = {}".format(self.runtime_path))
self.log.debug("Results path = {}".format(self.results_path))
self.log.debug("Temp path = {}".format(self.temp_path))
if len(self.user_conf_files) > 0:
self.log.debug("User defined configuration files:\n{}".format(
"\n".join(" {}".format(cf) for cf in self.user_conf_files)))
self.log.debug("Effective configuration: " + str(self.conf))
# Execute
try:
self.execute()
except Exception as ex:
self.log.exception(ex)
return -1
return 0
def run(self):
"Run the command and execute the command"
self.build_conf()
# Expand configuration variables
self.engine_conf = self.engine_conf_args.validated_conf()
self.case_conf = self.case_conf_args.validated_conf(expand_vars=False)
# Validate and process configuration
self.process_conf()
# Regenerate configuration
self.engine_conf = self.engine_conf_builder.get_conf()
self.case_conf = self.case_conf_builder.get_conf()
# Final logging configuration
log = logger.get_logger("")
log.removeHandler(log.handlers[0])
logging_conf = self.engine_conf.get("wok.logging")
logger.initialize(logging_conf)
# Show some debugging information
self.log.debug("Root path = {}".format(self.root_path))
self.log.debug("Conf path = {}".format(self.conf_path))
#self.log.debug("Runtime path = {}".format(self.runtime_path))
#self.log.debug("Results path = {}".format(self.results_path))
#self.log.debug("Temp path = {}".format(self.temp_path))
self.engine_conf_args.log_debug()
self.case_conf_args.log_debug()
# Execute
try:
self.execute()
except BaseException as ex:
self.log.error(str(ex))
from traceback import format_exc
self.log.debug(format_exc())
return -1
return 0
def _init_conf(self, app, conf_files, path_var, files_var, args_var):
self.logger.info("Checking Wok configuration files ...")
args = []
conf_path = app.config.get(path_var,
os.environ.get(path_var, os.getcwd()))
wok_conf_files = _get_conf_files(
conf_path, conf_files or [],
app.config.get(files_var, os.environ.get(files_var)), ".conf")
for path in wok_conf_files:
if not os.path.exists(path):
self.logger.error("--- {} (not found)".format(path))
continue
self.logger.info("+++ {}".format(path))
args += ["-c", path]
env_args = os.environ.get(args_var)
if env_args is not None:
env_args = env_args.strip().split(" ")
wok_conf_args = app.config.get(args_var, env_args)
if wok_conf_args is not None:
args += wok_conf_args
self.logger.debug("Arguments: {}".format(" ".join(wok_conf_args)))
self.logger.info("Loading Wok configuration ...")
parser = ArgumentParser()
wok_args = Arguments(parser, case_name_args=False, logger=self.logger)
wok_args.initialize(parser.parse_args(args))
self.conf_builder = wok_args.engine_conf_builder
self.conf = wok_args.engine_conf.expand_vars()
self.case_conf_builder = wok_args.case_conf_builder
self.case_conf = wok_args.case_conf.expand_vars()
# initialize logging according to the configuration
log = logger.get_logger("")
log.removeHandler(log.handlers[0])
logging_conf = self.conf.clone().expand_vars().get("wok.logging")
logger.initialize(logging_conf)
self.logger.debug(repr(self.conf))
def _init_conf(self, app, conf_files, path_var, files_var, args_var):
self.logger.info("Checking Wok configuration files ...")
args = []
conf_path = app.config.get(path_var, os.environ.get(path_var, os.getcwd()))
wok_conf_files = _get_conf_files(conf_path, conf_files or [],
app.config.get(files_var, os.environ.get(files_var)), ".conf")
for path in wok_conf_files:
if not os.path.exists(path):
self.logger.error("--- {} (not found)".format(path))
continue
self.logger.info("+++ {}".format(path))
args += ["-c", path]
env_args = os.environ.get(args_var)
if env_args is not None:
env_args = env_args.strip().split(" ")
wok_conf_args = app.config.get(args_var, env_args)
if wok_conf_args is not None:
args += wok_conf_args
self.logger.debug("Arguments: {}".format(" ".join(wok_conf_args)))
self.logger.info("Loading Wok configuration ...")
parser = ArgumentParser()
wok_args = Arguments(parser, case_name_args=False, logger=self.logger)
wok_args.initialize(parser.parse_args(args))
self.conf_builder = wok_args.engine_conf_builder
self.conf = wok_args.engine_conf.expand_vars()
self.case_conf_builder = wok_args.case_conf_builder
self.case_conf = wok_args.case_conf.expand_vars()
# initialize logging according to the configuration
log = logger.get_logger("")
log.removeHandler(log.handlers[0])
logging_conf = self.conf.clone().expand_vars().get("wok.logging")
logger.initialize(logging_conf)
self.logger.debug(repr(self.conf))
def __init__(self):
# Get task key and storage configuration
cmd_conf = OptionsConfig(
required=["case", "task", "index", "data.type", "storage.type"])
# Register signals
self._signal_names = {}
for signame in [x for x in dir(signal) if x.startswith("SIG")]:
try:
signum = getattr(signal, signame)
signal.signal(signum, self.__signal_handler)
self._signal_names[signum] = signame
except:
pass
# command line configuration
case_name = cmd_conf["case"]
task_cname = cmd_conf["task"]
workitem_index = cmd_conf["index"]
# initialize the data provider
provider_conf = cmd_conf["data"]
self._provider = data_provider_factory.create(provider_conf)
self._provider.start()
# initialize storage
storage_conf = cmd_conf["storage"]
self.storage = storage_factory.create(storage_conf)
self.storage = self.storage.get_container(case_name)
# load the module and task descriptors
task_desc = self._provider.load_task(case_name, task_cname)
workitem_desc = self._provider.load_workitem(case_name, task_cname,
workitem_index)
partition = workitem_desc["partition"]
# setup task configuration
self.conf = Data.create(task_desc["conf"])
self.conf["__task_index"] = workitem_index
self.conf.expand_vars()
# setup task attributes
self.case = workitem_desc["case"]
self.task = workitem_desc["task"]
self.id = workitem_desc["cname"]
self.name = workitem_desc["name"]
self.index = workitem_index
# initialize decorators
self._main = None
self._sources = []
self._foreach = None
self._begin = None
self._end = None
self._start_time = 0
self._end_time = self._start_time
# intialize task logging
log_conf = self.conf.get("logging")
logger.initialize(log_conf)
self.logger = logger.get_logger(self.name)
self.logger.debug("Task descriptor: {}".format(Data.create(task_desc)))
self.logger.debug("WorkItem descriptor: {}".format(
Data.create(workitem_desc)))
# Initialize input stream
self._stream = Stream(self._provider, task_desc["stream"])
# Initialize ports
self._ports = {}
self._in_ports = []
self._out_ports = []
self._open_ports = {}
if "ports" in task_desc and "ports" in partition:
port_descriptors = Data.create(task_desc["ports"])
for port_desc in port_descriptors.get("in", default=list):
port_desc["mode"] = PORT_MODE_IN
self._ports[port_desc["name"]] = port_desc
self._in_ports += [port_desc]
for port_desc in port_descriptors.get("out", default=list):
port_desc["mode"] = PORT_MODE_OUT
self._ports[port_desc["name"]] = port_desc
self._out_ports += [port_desc]
port_descriptors = Data.create(partition["ports"])
for port_desc in port_descriptors.get("in", default=list):
task_port_desc = self._ports[port_desc["name"]]
task_port_desc["data"] = port_desc["data"]
for port_desc in port_descriptors.get("out", default=list):
task_port_desc = self._ports[port_desc["name"]]
task_port_desc["data"] = port_desc["data"]
# Get hostname
try:
import socket
self.hostname = socket.gethostname()
except:
self.hostname = "unknown"
# The context field is free to be used by the task user to
# save variables related with the whole task life cycle.
# By default it is initialized with a dictionary but can be
# overwrote with any value by the user. Wok will never use it.
self.context = {}
def __init__(self):
# Get task key and storage configuration
cmd_conf = OptionsConfig(required=["case", "task", "index", "data.type", "storage.type"])
# Register signals
self._signal_names = {}
for signame in [x for x in dir(signal) if x.startswith("SIG")]:
try:
signum = getattr(signal, signame)
signal.signal(signum, self.__signal_handler)
self._signal_names[signum] = signame
except:
pass
# command line configuration
case_name = cmd_conf["case"]
task_cname = cmd_conf["task"]
workitem_index = cmd_conf["index"]
# initialize the data provider
provider_conf = cmd_conf["data"]
self._provider = data_provider_factory.create(provider_conf)
self._provider.start()
# initialize storage
storage_conf = cmd_conf["storage"]
self.storage = storage_factory.create(storage_conf)
self.storage = self.storage.get_container(case_name)
# load the module and task descriptors
task_desc = self._provider.load_task(case_name, task_cname)
workitem_desc = self._provider.load_workitem(case_name, task_cname, workitem_index)
partition = workitem_desc["partition"]
# setup task configuration
self.conf = Data.create(task_desc["conf"])
self.conf["__task_index"] = workitem_index
self.conf.expand_vars()
# setup task attributes
self.case = workitem_desc["case"]
self.task = workitem_desc["task"]
self.id = workitem_desc["cname"]
self.name = workitem_desc["name"]
self.index = workitem_index
# initialize decorators
self._main = None
self._sources = []
self._foreach = None
self._begin = None
self._end = None
self._start_time = 0
self._end_time = self._start_time
# intialize task logging
log_conf = self.conf.get("logging")
logger.initialize(log_conf)
self.logger = logger.get_logger(self.name)
self.logger.debug("Task descriptor: {}".format(Data.create(task_desc)))
self.logger.debug("WorkItem descriptor: {}".format(Data.create(workitem_desc)))
# Initialize input stream
self._stream = Stream(self._provider, task_desc["stream"])
# Initialize ports
self._ports = {}
self._in_ports = []
self._out_ports = []
self._open_ports = {}
if "ports" in task_desc and "ports" in partition:
port_descriptors = Data.create(task_desc["ports"])
for port_desc in port_descriptors.get("in", default=list):
port_desc["mode"] = PORT_MODE_IN
self._ports[port_desc["name"]] = port_desc
self._in_ports += [port_desc]
for port_desc in port_descriptors.get("out", default=list):
port_desc["mode"] = PORT_MODE_OUT
self._ports[port_desc["name"]] = port_desc
self._out_ports += [port_desc]
port_descriptors = Data.create(partition["ports"])
for port_desc in port_descriptors.get("in", default=list):
task_port_desc = self._ports[port_desc["name"]]
task_port_desc["data"] = port_desc["data"]
for port_desc in port_descriptors.get("out", default=list):
task_port_desc = self._ports[port_desc["name"]]
task_port_desc["data"] = port_desc["data"]
# Get hostname
try:
import socket
self.hostname = socket.gethostname()
except:
self.hostname = "unknown"
# The context field is free to be used by the task user to
# save variables related with the whole task life cycle.
# By default it is initialized with a dictionary but can be
# overwrote with any value by the user. Wok will never use it.
self.context = {}
def __init__(self):
# Get task key and storage configuration
cmd_conf = OptionsConfig(required=["instance_name", "module_path", "task_index", "storage.type"])
instance_name = cmd_conf["instance_name"]
module_path = cmd_conf["module_path"]
task_index = cmd_conf["task_index"]
storage_conf = cmd_conf["storage"]
storage = create_storage(storage_conf["type"], StorageContext.EXECUTION, storage_conf)
# Read data and configuration
self.data = storage.load_task_config(instance_name, module_path, task_index)
self.conf = self.data["conf"]
del self.data["conf"]
# set per task configuration values and expand
self.conf["__task_index"] = task_index
self.conf.expand_vars()
self.id = self.data["id"]
self.name = self.data["name"]
self.module_path = self.data["module"]
self.instance_name = self.data["instance"]
self._main = None
self._generators = []
self._foreach = None
self._begin = None
self._end = None
self._start_time = 0
self._end_time = self._start_time
logger.initialize(self.conf.get("log"))
self._log = self.logger()
# Initialize data iteration
iter_conf = self.data["iteration"]
self._iter_strategy = iter_conf.get("strategy")
self._iter_size = iter_conf.get("size", 0, dtype=int)
# Initialize ports
self._port_map = {}
self._in_ports = []
self._out_ports = []
if "ports" in self.data:
ports_conf = self.data["ports"]
for port_conf in ports_conf.get("in", []):
port = create_port(PORT_MODE_IN, port_conf, storage)
self._port_map[port.name] = port
self._in_ports += [port]
for port_conf in ports_conf.get("out", []):
port = create_port(PORT_MODE_OUT, port_conf, storage)
self._port_map[port.name] = port
self._out_ports += [port]
self.__ports_accessor = PortsAccessor(self._port_map)
# The context field is free to be used by the task user to
# save variables related with the whole task life cycle.
# By default it is initialized with a dictionary but can be
# overwrited with any value by the user. Wok will never use it.
self.context = {}
