def _create_platforms(self): """ Creates the platform according to the configuration :return: Platform """ platform_confs = self._conf.get("platforms") if platform_confs is None: platform_confs = Data.list() elif not Data.is_list(platform_confs): self._log.error("Wrong configuration type for 'platforms': {}".format(platform_confs)) platform_confs = Data.list() if len(platform_confs) == 0: platform_confs += [Data.element(dict(type="local"))] platforms = [] names = {} for pidx, platform_conf in enumerate(platform_confs): if isinstance(platform_conf, basestring): if not os.path.isabs(platform_conf) and self._conf_base_path is not None: platform_conf = os.path.join(self._conf_base_path, platform_conf) platform_conf = ConfigLoader(platform_conf).load() if not Data.is_element(platform_conf): raise errors.ConfigTypeError("wok.platforms[{}]".format(pidx, platform_conf)) ptype = platform_conf.get("type", "local") name = platform_conf.get("name", ptype) if name in names: name = "{}-{}".format(name, names[name]) names[name] += 1 else: names[name] = 2 platform_conf["name"] = name if "work_path" not in platform_conf: platform_conf["work_path"] = os.path.join(self._work_path, "platform_{}".format(name)) self._log.info("Creating '{}' platform ...".format(name)) self._log.debug("Platform configuration: {}".format(repr(platform_conf))) platforms += [create_platform(ptype, platform_conf)] return platforms
def _start(self): self._log.debug("Creating session ...") self._session = saga.Session() ctxs_conf = self._conf.get("context") if ctxs_conf is not None: if Data.is_element(ctxs_conf): ctxs_conf = Data.list([ctxs_conf]) for ctx_conf in ctxs_conf: try: ctx = saga.Context(ctx_conf["type"]) for key in ctx_conf: if hasattr(ctx, key): setattr(ctx, key, ctx_conf[key]) self._session.add_context(ctx) except Exception as ex: self._log.error("Wrong context configuration: {}".format(repr(ctx_conf))) self._log.exception(ex) self._log.debug("Creating job service ...") url = self._conf.get("service_url", "fork://localhost", dtype=str) self._job_service = saga.job.Service(url, session=self._session) self._remote_dir = saga.filesystem.Directory(self._file_url, session=self._session) # FIXME Use the logging configuration mechanisms of SAGA from wok import logger logger.init_logger("SGEJobService", conf=Data.element(dict(level=self._conf.get("saga_log.level", "error")))) # TODO count the number of previously queued jobs # TODO clean output files ? self._running = True self._run_thread = threading.Thread(target=self._run_handler, name="{}-run".format(self._name)) self._join_thread = threading.Thread(target=self._join_handler, name="{}-join".format(self._name)) self._run_thread.start() self._join_thread.start()
def initialize(conf=None, format=None, datefmt=None, level=None): """ Initialize the logging system. If conf is a dictionary then the parameters considered for configuration are: - format: Logger format - datefmt: Date format - loggers: list of tuples (name, conf) to configure loggers If conf is a list then only the loggers are configured. If conf is an string then the default logger is configured for the logging level. """ global _initialized if conf is None: conf = Data.element() elif not isinstance(conf, basestring): conf = Data.create(conf) if Data.is_list(conf): loggers_conf = conf conf = Data.element() elif Data.is_element(conf): loggers_conf = conf.get("loggers", default=Data.list) elif isinstance(conf, basestring): loggers_conf = Data.list([["", conf]]) conf = Data.element() format = format or conf.get("format", _DEFAULT_FORMAT) datefmt = datefmt or conf.get("datefmt", _DEFAULT_DATEFMT) logging.basicConfig(format=format, datefmt=datefmt) for (log_name, log_conf) in loggers_conf: init_logger(log_name, conf=log_conf) if level is not None: init_logger("", conf=level) _initialized = True
def init_logger(logger, conf): """ Initializa a logger from configuration. Configuration can be: - An string referring to the log level - A dictionary with the following parameters: - level: log level - handlers: List of log handlers or just a handler. Each handler have the following parameters: - type - ...: each handler type has a set of parameters Supported handlers: - smtp: Send logs by email. Parameters: - host - port (optional) - user - pass - from - to - subject - level - format: can be a simple string or a list of strings that will be joint with '\n' """ if isinstance(logger, basestring): logger = get_logger(logger) if isinstance(conf, basestring): conf = Data.element(dict(level=conf)) else: conf = Data.create(conf) level = conf.get("level") if level is not None: logger.setLevel(get_level(level)) handlers_conf = conf.get("handlers", default=Data.list) if Data.is_element(handlers_conf): handlers_conf = Data.list([handlers_conf]) for handler_conf in handlers_conf: handler = get_handler(logger, handler_conf) logger.addHandler(handler)
def prepare(self, case, task, index): execution = task.execution exec_conf = execution.conf if exec_conf is None: exec_conf = Data.element() if "script_path" not in exec_conf: raise MissingValueError("script_path") script_path = exec_conf["script_path"] lang = exec_conf.get("language", "python") case_conf = case.conf.clone().expand_vars() # Environment variables env = Data.element() #for k, v in os.environ.items(): # env[k] = v env.merge(task.conf.get(rtconf.TASK_ENV)) env.merge(exec_conf.get("env")) # Default module script path platform_project_path = task.conf.get(rtconf.PROJECT_PATH, case.project.path) flow_path = os.path.abspath(os.path.dirname(task.flow_path)) flow_rel_path = os.path.relpath(flow_path, case.project.path) platform_script_path = os.path.join(platform_project_path, flow_rel_path, script_path) env[ENV_PROJECT_PATH] = platform_project_path env[ENV_FLOW_PATH] = flow_rel_path env[ENV_SCRIPT_PATH] = script_path env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path script = [] sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list) if isinstance(sources, basestring): sources = Data.list([sources]) for source in sources: script += ['source "{}"'.format(source)] if lang == "python": virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV) if virtualenv is not None: #script += ["set -x"] #script += ["echo Activating virtualenv {} ...".format(virtualenv)] script += ['source "{}"'.format(os.path.join(virtualenv, "bin", "activate"))] #script += ["set +x"] #script += ["echo Running workitem ..."] cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")] cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)] lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS) if lib_path is not None: if Data.is_list(lib_path): lib_path = ":".join(lib_path) if "PYTHONPATH" in env: env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"] else: env["PYTHONPATH"] = lib_path else: raise LanguageError(lang) cmd += ["-D", "case={}".format(case.name), "-D", "task={}".format(task.cname), "-D", "index={}".format(index)] #for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf): # cmd += ["-D", "storage.{}={}".format(key, value)] for key, value in self._plain_conf(Data.create(task.platform.data.context_conf(CTX_EXEC))): cmd += ["-D", "data.{}={}".format(key, value)] for key, value in self._plain_conf(task.platform.storage.context_conf(CTX_EXEC)): cmd += ["-D", "storage.{}={}".format(key, value)] script += [" ".join(cmd)] return "\n".join(script), env.to_native()
class NativeCommmandBuilder(CommmandBuilder): def _plain_conf(self, value, path=None): if path is None: path = [] if not Data.is_element(value): yield (".".join(path), value) else: for key in value.keys(): for k, v in self._plain_conf(value[key], path + [key]): yield (k, v) def prepare(self, case, task, index): execution = task.execution exec_conf = execution.conf if exec_conf is None: exec_conf = Data.element() if "script_path" not in exec_conf: raise MissingValueError("script_path") script_path = exec_conf["script_path"] lang = exec_conf.get("language", "python") case_conf = case.conf.clone().expand_vars() # Environment variables env = Data.element() #for k, v in os.environ.items(): # env[k] = v env.merge(task.conf.get(rtconf.TASK_ENV)) env.merge(exec_conf.get("env")) # Default module script path platform_project_path = task.conf.get(rtconf.PROJECT_PATH, case.project.path) flow_path = os.path.abspath(os.path.dirname(task.flow_path)) flow_rel_path = os.path.relpath(flow_path, case.project.path) platform_script_path = os.path.join(platform_project_path, flow_rel_path, script_path) env[ENV_PROJECT_PATH] = platform_project_path env[ENV_FLOW_PATH] = flow_rel_path env[ENV_SCRIPT_PATH] = script_path env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path script = [] sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list) if isinstance(sources, basestring): sources = Data.list([sources]) for source in sources: script += ['source "{}"'.format(source)] if lang == "python": virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV) if virtualenv is not None: #script += ["set -x"] #script += ["echo Activating virtualenv {} ...".format(virtualenv)] script += [ 'source "{}"'.format( os.path.join(virtualenv, "bin", "activate")) ] #script += ["set +x"] #script += ["echo Running workitem ..."] cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")] cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)] lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS) if lib_path is not None: if Data.is_list(lib_path): lib_path = ":".join(lib_path) if "PYTHONPATH" in env: env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"] else: env["PYTHONPATH"] = lib_path else: raise LanguageError(lang) cmd += [ "-D", "case={}".format(case.name), "-D", "task={}".format(task.cname), "-D", "index={}".format(index) ] #for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf): # cmd += ["-D", "storage.{}={}".format(key, value)] for key, value in self._plain_conf( Data.create(task.platform.data.context_conf(CTX_EXEC))): cmd += ["-D", "data.{}={}".format(key, value)] for key, value in self._plain_conf( task.platform.storage.context_conf(CTX_EXEC)): cmd += ["-D", "storage.{}={}".format(key, value)] script += [" ".join(cmd)] return "\n".join(script), env.to_native()