コード例 #1
0
ファイル: engine.py プロジェクト: bbglab/wok
	def _create_platforms(self):
		"""
		Creates the platform according to the configuration
		:return: Platform
		"""

		platform_confs = self._conf.get("platforms")
		if platform_confs is None:
			platform_confs = Data.list()
		elif not Data.is_list(platform_confs):
			self._log.error("Wrong configuration type for 'platforms': {}".format(platform_confs))
			platform_confs = Data.list()

		if len(platform_confs) == 0:
			platform_confs += [Data.element(dict(type="local"))]

		platforms = []

		names = {}
		for pidx, platform_conf in enumerate(platform_confs):
			if isinstance(platform_conf, basestring):
				if not os.path.isabs(platform_conf) and self._conf_base_path is not None:
					platform_conf = os.path.join(self._conf_base_path, platform_conf)
				platform_conf = ConfigLoader(platform_conf).load()

			if not Data.is_element(platform_conf):
				raise errors.ConfigTypeError("wok.platforms[{}]".format(pidx, platform_conf))

			ptype = platform_conf.get("type", "local")

			name = platform_conf.get("name", ptype)
			if name in names:
				name = "{}-{}".format(name, names[name])
				names[name] += 1
			else:
				names[name] = 2
			platform_conf["name"] = name

			if "work_path" not in platform_conf:
				platform_conf["work_path"] = os.path.join(self._work_path, "platform_{}".format(name))

			self._log.info("Creating '{}' platform ...".format(name))
			self._log.debug("Platform configuration: {}".format(repr(platform_conf)))

			platforms += [create_platform(ptype, platform_conf)]

		return platforms
コード例 #2
0
	def _start(self):

		self._log.debug("Creating session ...")

		self._session = saga.Session()

		ctxs_conf = self._conf.get("context")
		if ctxs_conf is not None:
			if Data.is_element(ctxs_conf):
				ctxs_conf = Data.list([ctxs_conf])

			for ctx_conf in ctxs_conf:
				try:
					ctx = saga.Context(ctx_conf["type"])
					for key in ctx_conf:
						if hasattr(ctx, key):
							setattr(ctx, key, ctx_conf[key])
					self._session.add_context(ctx)
				except Exception as ex:
					self._log.error("Wrong context configuration: {}".format(repr(ctx_conf)))
					self._log.exception(ex)

		self._log.debug("Creating job service ...")

		url = self._conf.get("service_url", "fork://localhost", dtype=str)
		self._job_service = saga.job.Service(url, session=self._session)

		self._remote_dir = saga.filesystem.Directory(self._file_url, session=self._session)

		# FIXME Use the logging configuration mechanisms of SAGA
		from wok import logger
		logger.init_logger("SGEJobService", conf=Data.element(dict(level=self._conf.get("saga_log.level", "error"))))

		# TODO count the number of previously queued jobs

		# TODO clean output files ?

		self._running = True
		self._run_thread = threading.Thread(target=self._run_handler, name="{}-run".format(self._name))
		self._join_thread = threading.Thread(target=self._join_handler, name="{}-join".format(self._name))
		self._run_thread.start()
		self._join_thread.start()
コード例 #3
0
ファイル: logger.py プロジェクト: bbglab/wok
def initialize(conf=None, format=None, datefmt=None, level=None):
	"""
	Initialize the logging system.

	If conf is a dictionary then the parameters considered for configuration are:
	  - format: Logger format
	  - datefmt: Date format
	  - loggers: list of tuples (name, conf) to configure loggers

	If conf is a list then only the loggers are configured.

	If conf is an string then the default logger is configured for the logging level.
	"""

	global _initialized

	if conf is None:
		conf = Data.element()
	elif not isinstance(conf, basestring):
		conf = Data.create(conf)

	if Data.is_list(conf):
		loggers_conf = conf
		conf = Data.element()
	elif Data.is_element(conf):
		loggers_conf = conf.get("loggers", default=Data.list)
	elif isinstance(conf, basestring):
		loggers_conf = Data.list([["", conf]])
		conf = Data.element()

	format = format or conf.get("format", _DEFAULT_FORMAT)
	datefmt = datefmt or conf.get("datefmt", _DEFAULT_DATEFMT)

	logging.basicConfig(format=format, datefmt=datefmt)

	for (log_name, log_conf) in loggers_conf:
		init_logger(log_name, conf=log_conf)

	if level is not None:
		init_logger("", conf=level)

	_initialized = True
コード例 #4
0
ファイル: logger.py プロジェクト: philloidin/wok
def initialize(conf=None, format=None, datefmt=None, level=None):
    """
	Initialize the logging system.

	If conf is a dictionary then the parameters considered for configuration are:
	  - format: Logger format
	  - datefmt: Date format
	  - loggers: list of tuples (name, conf) to configure loggers

	If conf is a list then only the loggers are configured.

	If conf is an string then the default logger is configured for the logging level.
	"""

    global _initialized

    if conf is None:
        conf = Data.element()
    elif not isinstance(conf, basestring):
        conf = Data.create(conf)

    if Data.is_list(conf):
        loggers_conf = conf
        conf = Data.element()
    elif Data.is_element(conf):
        loggers_conf = conf.get("loggers", default=Data.list)
    elif isinstance(conf, basestring):
        loggers_conf = Data.list([["", conf]])
        conf = Data.element()

    format = format or conf.get("format", _DEFAULT_FORMAT)
    datefmt = datefmt or conf.get("datefmt", _DEFAULT_DATEFMT)

    logging.basicConfig(format=format, datefmt=datefmt)

    for (log_name, log_conf) in loggers_conf:
        init_logger(log_name, conf=log_conf)

    if level is not None:
        init_logger("", conf=level)

    _initialized = True
コード例 #5
0
ファイル: logger.py プロジェクト: bbglab/wok
def init_logger(logger, conf):
	"""
	Initializa a logger from configuration. Configuration can be:
	- An string referring to the log level
	- A dictionary with the following parameters:
	  - level: log level
	  - handlers: List of log handlers or just a handler. Each handler have the following parameters:
	    - type
	    - ...: each handler type has a set of parameters

	Supported handlers:
	- smtp: Send logs by email. Parameters:
	  - host
	  - port (optional)
	  - user
	  - pass
	  - from
	  - to
	  - subject
	  - level
	  - format: can be a simple string or a list of strings that will be joint with '\n'
	"""
	if isinstance(logger, basestring):
		logger = get_logger(logger)

	if isinstance(conf, basestring):
		conf = Data.element(dict(level=conf))
	else:
		conf = Data.create(conf)

	level = conf.get("level")
	if level is not None:
		logger.setLevel(get_level(level))

	handlers_conf = conf.get("handlers", default=Data.list)
	if Data.is_element(handlers_conf):
		handlers_conf = Data.list([handlers_conf])

	for handler_conf in handlers_conf:
		handler = get_handler(logger, handler_conf)
		logger.addHandler(handler)
コード例 #6
0
ファイル: logger.py プロジェクト: philloidin/wok
def init_logger(logger, conf):
    """
	Initializa a logger from configuration. Configuration can be:
	- An string referring to the log level
	- A dictionary with the following parameters:
	  - level: log level
	  - handlers: List of log handlers or just a handler. Each handler have the following parameters:
	    - type
	    - ...: each handler type has a set of parameters

	Supported handlers:
	- smtp: Send logs by email. Parameters:
	  - host
	  - port (optional)
	  - user
	  - pass
	  - from
	  - to
	  - subject
	  - level
	  - format: can be a simple string or a list of strings that will be joint with '\n'
	"""
    if isinstance(logger, basestring):
        logger = get_logger(logger)

    if isinstance(conf, basestring):
        conf = Data.element(dict(level=conf))
    else:
        conf = Data.create(conf)

    level = conf.get("level")
    if level is not None:
        logger.setLevel(get_level(level))

    handlers_conf = conf.get("handlers", default=Data.list)
    if Data.is_element(handlers_conf):
        handlers_conf = Data.list([handlers_conf])

    for handler_conf in handlers_conf:
        handler = get_handler(logger, handler_conf)
        logger.addHandler(handler)
コード例 #7
0
ファイル: native.py プロジェクト: bbglab/wok
	def prepare(self, case, task, index):
		execution = task.execution
		exec_conf = execution.conf
		if exec_conf is None:
			exec_conf = Data.element()

		if "script_path" not in exec_conf:
			raise MissingValueError("script_path")

		script_path = exec_conf["script_path"]

		lang = exec_conf.get("language", "python")

		case_conf = case.conf.clone().expand_vars()

		# Environment variables
		env = Data.element()
		#for k, v in os.environ.items():
		#	env[k] = v
		env.merge(task.conf.get(rtconf.TASK_ENV))
		env.merge(exec_conf.get("env"))

		# Default module script path
		platform_project_path = task.conf.get(rtconf.PROJECT_PATH, case.project.path)
		flow_path = os.path.abspath(os.path.dirname(task.flow_path))
		flow_rel_path = os.path.relpath(flow_path, case.project.path)
		platform_script_path = os.path.join(platform_project_path, flow_rel_path, script_path)
		env[ENV_PROJECT_PATH] = platform_project_path
		env[ENV_FLOW_PATH] = flow_rel_path
		env[ENV_SCRIPT_PATH] = script_path
		env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path

		script = []
		
		sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list)
		if isinstance(sources, basestring):
			sources = Data.list([sources])

		for source in sources:
			script += ['source "{}"'.format(source)]
	
		if lang == "python":
			virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV)
			if virtualenv is not None:
				#script += ["set -x"]
				#script += ["echo Activating virtualenv {} ...".format(virtualenv)]
				script += ['source "{}"'.format(os.path.join(virtualenv, "bin", "activate"))]
				#script += ["set +x"]

			#script += ["echo Running workitem ..."]

			cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")]
			cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)]

			lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS)
			if lib_path is not None:
				if Data.is_list(lib_path):
					lib_path = ":".join(lib_path)

				if "PYTHONPATH" in env:
					env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"]
				else:
					env["PYTHONPATH"] = lib_path
		else:
			raise LanguageError(lang)

		cmd += ["-D", "case={}".format(case.name),
				"-D", "task={}".format(task.cname),
				"-D", "index={}".format(index)]

		#for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf):
		#	cmd += ["-D", "storage.{}={}".format(key, value)]

		for key, value in self._plain_conf(Data.create(task.platform.data.context_conf(CTX_EXEC))):
			cmd += ["-D", "data.{}={}".format(key, value)]

		for key, value in self._plain_conf(task.platform.storage.context_conf(CTX_EXEC)):
			cmd += ["-D", "storage.{}={}".format(key, value)]

		script += [" ".join(cmd)]
		
		return "\n".join(script), env.to_native()
コード例 #8
0
class NativeCommmandBuilder(CommmandBuilder):
    def _plain_conf(self, value, path=None):
        if path is None:
            path = []

        if not Data.is_element(value):
            yield (".".join(path), value)
        else:
            for key in value.keys():
                for k, v in self._plain_conf(value[key], path + [key]):
                    yield (k, v)

    def prepare(self, case, task, index):
        execution = task.execution
        exec_conf = execution.conf
        if exec_conf is None:
            exec_conf = Data.element()

        if "script_path" not in exec_conf:
            raise MissingValueError("script_path")

        script_path = exec_conf["script_path"]

        lang = exec_conf.get("language", "python")

        case_conf = case.conf.clone().expand_vars()

        # Environment variables
        env = Data.element()
        #for k, v in os.environ.items():
        #	env[k] = v
        env.merge(task.conf.get(rtconf.TASK_ENV))
        env.merge(exec_conf.get("env"))

        # Default module script path
        platform_project_path = task.conf.get(rtconf.PROJECT_PATH,
                                              case.project.path)
        flow_path = os.path.abspath(os.path.dirname(task.flow_path))
        flow_rel_path = os.path.relpath(flow_path, case.project.path)
        platform_script_path = os.path.join(platform_project_path,
                                            flow_rel_path, script_path)
        env[ENV_PROJECT_PATH] = platform_project_path
        env[ENV_FLOW_PATH] = flow_rel_path
        env[ENV_SCRIPT_PATH] = script_path
        env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path

        script = []

        sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list)
        if isinstance(sources, basestring):
            sources = Data.list([sources])

        for source in sources:
            script += ['source "{}"'.format(source)]

        if lang == "python":
            virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV)
            if virtualenv is not None:
                #script += ["set -x"]
                #script += ["echo Activating virtualenv {} ...".format(virtualenv)]
                script += [
                    'source "{}"'.format(
                        os.path.join(virtualenv, "bin", "activate"))
                ]
                #script += ["set +x"]

            #script += ["echo Running workitem ..."]

            cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")]
            cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)]

            lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS)
            if lib_path is not None:
                if Data.is_list(lib_path):
                    lib_path = ":".join(lib_path)

                if "PYTHONPATH" in env:
                    env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"]
                else:
                    env["PYTHONPATH"] = lib_path
        else:
            raise LanguageError(lang)

        cmd += [
            "-D", "case={}".format(case.name), "-D",
            "task={}".format(task.cname), "-D", "index={}".format(index)
        ]

        #for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf):
        #	cmd += ["-D", "storage.{}={}".format(key, value)]

        for key, value in self._plain_conf(
                Data.create(task.platform.data.context_conf(CTX_EXEC))):
            cmd += ["-D", "data.{}={}".format(key, value)]

        for key, value in self._plain_conf(
                task.platform.storage.context_conf(CTX_EXEC)):
            cmd += ["-D", "storage.{}={}".format(key, value)]

        script += [" ".join(cmd)]

        return "\n".join(script), env.to_native()