def _create_platforms(self):
"""
Creates the platform according to the configuration
:return: Platform
"""
platform_confs = self._conf.get("platforms")
if platform_confs is None:
platform_confs = Data.list()
elif not Data.is_list(platform_confs):
self._log.error("Wrong configuration type for 'platforms': {}".format(platform_confs))
platform_confs = Data.list()
if len(platform_confs) == 0:
platform_confs += [Data.element(dict(type="local"))]
platforms = []
names = {}
for pidx, platform_conf in enumerate(platform_confs):
if isinstance(platform_conf, basestring):
if not os.path.isabs(platform_conf) and self._conf_base_path is not None:
platform_conf = os.path.join(self._conf_base_path, platform_conf)
platform_conf = ConfigLoader(platform_conf).load()
if not Data.is_element(platform_conf):
raise errors.ConfigTypeError("wok.platforms[{}]".format(pidx, platform_conf))
ptype = platform_conf.get("type", "local")
name = platform_conf.get("name", ptype)
if name in names:
name = "{}-{}".format(name, names[name])
names[name] += 1
else:
names[name] = 2
platform_conf["name"] = name
if "work_path" not in platform_conf:
platform_conf["work_path"] = os.path.join(self._work_path, "platform_{}".format(name))
self._log.info("Creating '{}' platform ...".format(name))
self._log.debug("Platform configuration: {}".format(repr(platform_conf)))
platforms += [create_platform(ptype, platform_conf)]
return platforms
def _start(self):
self._log.debug("Creating session ...")
self._session = saga.Session()
ctxs_conf = self._conf.get("context")
if ctxs_conf is not None:
if Data.is_element(ctxs_conf):
ctxs_conf = Data.list([ctxs_conf])
for ctx_conf in ctxs_conf:
try:
ctx = saga.Context(ctx_conf["type"])
for key in ctx_conf:
if hasattr(ctx, key):
setattr(ctx, key, ctx_conf[key])
self._session.add_context(ctx)
except Exception as ex:
self._log.error("Wrong context configuration: {}".format(repr(ctx_conf)))
self._log.exception(ex)
self._log.debug("Creating job service ...")
url = self._conf.get("service_url", "fork://localhost", dtype=str)
self._job_service = saga.job.Service(url, session=self._session)
self._remote_dir = saga.filesystem.Directory(self._file_url, session=self._session)
# FIXME Use the logging configuration mechanisms of SAGA
from wok import logger
logger.init_logger("SGEJobService", conf=Data.element(dict(level=self._conf.get("saga_log.level", "error"))))
# TODO count the number of previously queued jobs
# TODO clean output files ?
self._running = True
self._run_thread = threading.Thread(target=self._run_handler, name="{}-run".format(self._name))
self._join_thread = threading.Thread(target=self._join_handler, name="{}-join".format(self._name))
self._run_thread.start()
self._join_thread.start()
def initialize(conf=None, format=None, datefmt=None, level=None):
"""
Initialize the logging system.
If conf is a dictionary then the parameters considered for configuration are:
- format: Logger format
- datefmt: Date format
- loggers: list of tuples (name, conf) to configure loggers
If conf is a list then only the loggers are configured.
If conf is an string then the default logger is configured for the logging level.
"""
global _initialized
if conf is None:
conf = Data.element()
elif not isinstance(conf, basestring):
conf = Data.create(conf)
if Data.is_list(conf):
loggers_conf = conf
conf = Data.element()
elif Data.is_element(conf):
loggers_conf = conf.get("loggers", default=Data.list)
elif isinstance(conf, basestring):
loggers_conf = Data.list([["", conf]])
conf = Data.element()
format = format or conf.get("format", _DEFAULT_FORMAT)
datefmt = datefmt or conf.get("datefmt", _DEFAULT_DATEFMT)
logging.basicConfig(format=format, datefmt=datefmt)
for (log_name, log_conf) in loggers_conf:
init_logger(log_name, conf=log_conf)
if level is not None:
init_logger("", conf=level)
_initialized = True
def initialize(conf=None, format=None, datefmt=None, level=None):
"""
Initialize the logging system.
If conf is a dictionary then the parameters considered for configuration are:
- format: Logger format
- datefmt: Date format
- loggers: list of tuples (name, conf) to configure loggers
If conf is a list then only the loggers are configured.
If conf is an string then the default logger is configured for the logging level.
"""
global _initialized
if conf is None:
conf = Data.element()
elif not isinstance(conf, basestring):
conf = Data.create(conf)
if Data.is_list(conf):
loggers_conf = conf
conf = Data.element()
elif Data.is_element(conf):
loggers_conf = conf.get("loggers", default=Data.list)
elif isinstance(conf, basestring):
loggers_conf = Data.list([["", conf]])
conf = Data.element()
format = format or conf.get("format", _DEFAULT_FORMAT)
datefmt = datefmt or conf.get("datefmt", _DEFAULT_DATEFMT)
logging.basicConfig(format=format, datefmt=datefmt)
for (log_name, log_conf) in loggers_conf:
init_logger(log_name, conf=log_conf)
if level is not None:
init_logger("", conf=level)
_initialized = True
def init_logger(logger, conf):
"""
Initializa a logger from configuration. Configuration can be:
- An string referring to the log level
- A dictionary with the following parameters:
- level: log level
- handlers: List of log handlers or just a handler. Each handler have the following parameters:
- type
- ...: each handler type has a set of parameters
Supported handlers:
- smtp: Send logs by email. Parameters:
- host
- port (optional)
- user
- pass
- from
- to
- subject
- level
- format: can be a simple string or a list of strings that will be joint with '\n'
"""
if isinstance(logger, basestring):
logger = get_logger(logger)
if isinstance(conf, basestring):
conf = Data.element(dict(level=conf))
else:
conf = Data.create(conf)
level = conf.get("level")
if level is not None:
logger.setLevel(get_level(level))
handlers_conf = conf.get("handlers", default=Data.list)
if Data.is_element(handlers_conf):
handlers_conf = Data.list([handlers_conf])
for handler_conf in handlers_conf:
handler = get_handler(logger, handler_conf)
logger.addHandler(handler)
def init_logger(logger, conf):
"""
Initializa a logger from configuration. Configuration can be:
- An string referring to the log level
- A dictionary with the following parameters:
- level: log level
- handlers: List of log handlers or just a handler. Each handler have the following parameters:
- type
- ...: each handler type has a set of parameters
Supported handlers:
- smtp: Send logs by email. Parameters:
- host
- port (optional)
- user
- pass
- from
- to
- subject
- level
- format: can be a simple string or a list of strings that will be joint with '\n'
"""
if isinstance(logger, basestring):
logger = get_logger(logger)
if isinstance(conf, basestring):
conf = Data.element(dict(level=conf))
else:
conf = Data.create(conf)
level = conf.get("level")
if level is not None:
logger.setLevel(get_level(level))
handlers_conf = conf.get("handlers", default=Data.list)
if Data.is_element(handlers_conf):
handlers_conf = Data.list([handlers_conf])
for handler_conf in handlers_conf:
handler = get_handler(logger, handler_conf)
logger.addHandler(handler)
def prepare(self, case, task, index):
execution = task.execution
exec_conf = execution.conf
if exec_conf is None:
exec_conf = Data.element()
if "script_path" not in exec_conf:
raise MissingValueError("script_path")
script_path = exec_conf["script_path"]
lang = exec_conf.get("language", "python")
case_conf = case.conf.clone().expand_vars()
# Environment variables
env = Data.element()
#for k, v in os.environ.items():
# env[k] = v
env.merge(task.conf.get(rtconf.TASK_ENV))
env.merge(exec_conf.get("env"))
# Default module script path
platform_project_path = task.conf.get(rtconf.PROJECT_PATH, case.project.path)
flow_path = os.path.abspath(os.path.dirname(task.flow_path))
flow_rel_path = os.path.relpath(flow_path, case.project.path)
platform_script_path = os.path.join(platform_project_path, flow_rel_path, script_path)
env[ENV_PROJECT_PATH] = platform_project_path
env[ENV_FLOW_PATH] = flow_rel_path
env[ENV_SCRIPT_PATH] = script_path
env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path
script = []
sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list)
if isinstance(sources, basestring):
sources = Data.list([sources])
for source in sources:
script += ['source "{}"'.format(source)]
if lang == "python":
virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV)
if virtualenv is not None:
#script += ["set -x"]
#script += ["echo Activating virtualenv {} ...".format(virtualenv)]
script += ['source "{}"'.format(os.path.join(virtualenv, "bin", "activate"))]
#script += ["set +x"]
#script += ["echo Running workitem ..."]
cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")]
cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)]
lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS)
if lib_path is not None:
if Data.is_list(lib_path):
lib_path = ":".join(lib_path)
if "PYTHONPATH" in env:
env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"]
else:
env["PYTHONPATH"] = lib_path
else:
raise LanguageError(lang)
cmd += ["-D", "case={}".format(case.name),
"-D", "task={}".format(task.cname),
"-D", "index={}".format(index)]
#for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf):
# cmd += ["-D", "storage.{}={}".format(key, value)]
for key, value in self._plain_conf(Data.create(task.platform.data.context_conf(CTX_EXEC))):
cmd += ["-D", "data.{}={}".format(key, value)]
for key, value in self._plain_conf(task.platform.storage.context_conf(CTX_EXEC)):
cmd += ["-D", "storage.{}={}".format(key, value)]
script += [" ".join(cmd)]
return "\n".join(script), env.to_native()
class NativeCommmandBuilder(CommmandBuilder):
def _plain_conf(self, value, path=None):
if path is None:
path = []
if not Data.is_element(value):
yield (".".join(path), value)
else:
for key in value.keys():
for k, v in self._plain_conf(value[key], path + [key]):
yield (k, v)
def prepare(self, case, task, index):
execution = task.execution
exec_conf = execution.conf
if exec_conf is None:
exec_conf = Data.element()
if "script_path" not in exec_conf:
raise MissingValueError("script_path")
script_path = exec_conf["script_path"]
lang = exec_conf.get("language", "python")
case_conf = case.conf.clone().expand_vars()
# Environment variables
env = Data.element()
#for k, v in os.environ.items():
# env[k] = v
env.merge(task.conf.get(rtconf.TASK_ENV))
env.merge(exec_conf.get("env"))
# Default module script path
platform_project_path = task.conf.get(rtconf.PROJECT_PATH,
case.project.path)
flow_path = os.path.abspath(os.path.dirname(task.flow_path))
flow_rel_path = os.path.relpath(flow_path, case.project.path)
platform_script_path = os.path.join(platform_project_path,
flow_rel_path, script_path)
env[ENV_PROJECT_PATH] = platform_project_path
env[ENV_FLOW_PATH] = flow_rel_path
env[ENV_SCRIPT_PATH] = script_path
env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path
script = []
sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list)
if isinstance(sources, basestring):
sources = Data.list([sources])
for source in sources:
script += ['source "{}"'.format(source)]
if lang == "python":
virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV)
if virtualenv is not None:
#script += ["set -x"]
#script += ["echo Activating virtualenv {} ...".format(virtualenv)]
script += [
'source "{}"'.format(
os.path.join(virtualenv, "bin", "activate"))
]
#script += ["set +x"]
#script += ["echo Running workitem ..."]
cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")]
cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)]
lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS)
if lib_path is not None:
if Data.is_list(lib_path):
lib_path = ":".join(lib_path)
if "PYTHONPATH" in env:
env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"]
else:
env["PYTHONPATH"] = lib_path
else:
raise LanguageError(lang)
cmd += [
"-D", "case={}".format(case.name), "-D",
"task={}".format(task.cname), "-D", "index={}".format(index)
]
#for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf):
# cmd += ["-D", "storage.{}={}".format(key, value)]
for key, value in self._plain_conf(
Data.create(task.platform.data.context_conf(CTX_EXEC))):
cmd += ["-D", "data.{}={}".format(key, value)]
for key, value in self._plain_conf(
task.platform.storage.context_conf(CTX_EXEC)):
cmd += ["-D", "storage.{}={}".format(key, value)]
script += [" ".join(cmd)]
return "\n".join(script), env.to_native()
