コード例 #1
0
ファイル: start.py プロジェクト: bbglab/wok
def main():
	conf = OptionsConfig()

	if "wok" not in conf:
		print("Missing wok configuration")
		exit(-1)

	conf.expand_vars()

	logger.initialize(conf.get("wok.logging"))

	log = logger.get_logger("wok.server.start")

	# run the server

	retcode = 0
	try:
		server = WokServer(conf, app)
		server.init()
		server.run()
	except Exception as e:
		log.exception(e)
		retcode = -1

	exit(retcode)
コード例 #2
0
ファイル: start.py プロジェクト: philloidin/wok
def main():
    conf = OptionsConfig()

    if "wok" not in conf:
        print("Missing wok configuration")
        exit(-1)

    conf.expand_vars()

    logger.initialize(conf.get("wok.logging"))

    log = logger.get_logger("wok.server.start")

    # run the server

    retcode = 0
    try:
        server = WokServer(conf, app)
        server.init()
        server.run()
    except Exception as e:
        log.exception(e)
        retcode = -1

    exit(retcode)
コード例 #3
0
ファイル: command.py プロジェクト: chris-zen/phd-thesis
	def __init__(self, args_usage="", epilog="", logger_name=None):

		self.args_usage = args_usage
		self.epilog = epilog

		if logger_name is None:
			if hasattr(self, "cmd_name"):
				logger_name = self.cmd_name
			else:
				logger_name = os.path.splitext(os.path.basename(sys.argv[0]))[0]

		# Determine binary path

		script_path = os.path.dirname(sys.argv[0])
		if script_path:
			self.launchers_path = os.path.abspath(script_path)
		else:
			self.launchers_path = os.getcwd()

		# Determine default paths

		self.root_path = os.path.normpath(os.path.join(self.launchers_path, ".."))

		self.bin_path = os.path.join(self.root_path, "bin")

		self.data_path = os.path.join(self.root_path, "data")

		self.conf_path = os.path.join(self.root_path, "conf")

		self.workflows_path = os.path.join(self.root_path, "workflows")

		self.runtime_path = os.path.join(self.root_path, "runtime")

		# Parse arguments

		parser = ArgumentParser(
			prog = "run " + self.cmd_name,
			epilog = self.epilog,
			formatter_class=RawDescriptionHelpFormatter)

		parser.add_argument('--version', action='version', version='%(prog)s ' + VERSION)

		self.add_arguments(parser)

		parser.add_argument("-L", "--log-level", dest="log_level",
						  default=None, choices=["debug", "info", "warn", "error", "critical", "notset"],
						  help="Define the logging level")

		self.args = parser.parse_args()

		# Create preliminary logger

		level = self.args.log_level or "info"
		logger.initialize(level=level)


		self.log = logger.get_logger(logger_name, level=level)
コード例 #4
0
def main():
	conf = Config()

	initialize({"log":{"level":"debug"}})
	log = get_logger("mrna_preproc_counts")
	log.setLevel(logging.DEBUG)

	log.info("Querying log2r tumour units ...")

	es = EntityServer(conf["entities"])
	em = es.manager()
コード例 #5
0
ファイル: command.py プロジェクト: chris-zen/phd-thesis
	def run(self):
		"Run the command and execute the command"

		self.build_conf()

		# Expand configuration variables

		self.conf = self.conf_builder.get_conf()
		
		self.conf.expand_vars()
		#self.log.debug(repr(self.conf))

		# Validate and process configuration

		self.process_conf()
		
		# Regenerate configuration
		
		self.conf = self.conf_builder.get_conf()

		# Final logging configuration

		log = logger.get_logger("")
		log.removeHandler(log.handlers[0])
		logging_conf = self.conf.get("wok.logging")
		logger.initialize(logging_conf)

		# Show some debugging information

		self.log.debug("Root path = {}".format(self.root_path))
		self.log.debug("Conf path = {}".format(self.conf_path))
		self.log.debug("Data path = {}".format(self.data_path))
		self.log.debug("Workflows path = {}".format(self.workflows_path))
		self.log.debug("Runtime path = {}".format(self.runtime_path))
		self.log.debug("Results path = {}".format(self.results_path))
		self.log.debug("Temp path = {}".format(self.temp_path))

		if len(self.user_conf_files) > 0:
			self.log.debug("User defined configuration files:\n{}".format(
							"\n".join("  {}".format(cf) for cf in self.user_conf_files)))

		self.log.debug("Effective configuration: " + str(self.conf))
		
		# Execute

		try:
			self.execute()
		except Exception as ex:
			self.log.exception(ex)
			return -1

		return 0
コード例 #6
0
ファイル: command.py プロジェクト: chris-zen/phd-thesis
	def run(self):
		"Run the command and execute the command"

		self.build_conf()

		# Expand configuration variables

		self.engine_conf = self.engine_conf_args.validated_conf()
		self.case_conf = self.case_conf_args.validated_conf(expand_vars=False)

		# Validate and process configuration

		self.process_conf()
		
		# Regenerate configuration

		self.engine_conf = self.engine_conf_builder.get_conf()
		self.case_conf = self.case_conf_builder.get_conf()

		# Final logging configuration

		log = logger.get_logger("")
		log.removeHandler(log.handlers[0])
		logging_conf = self.engine_conf.get("wok.logging")
		logger.initialize(logging_conf)

		# Show some debugging information

		self.log.debug("Root path = {}".format(self.root_path))
		self.log.debug("Conf path = {}".format(self.conf_path))
		#self.log.debug("Runtime path = {}".format(self.runtime_path))
		#self.log.debug("Results path = {}".format(self.results_path))
		#self.log.debug("Temp path = {}".format(self.temp_path))

		self.engine_conf_args.log_debug()
		self.case_conf_args.log_debug()

		# Execute

		try:
			self.execute()
		except BaseException as ex:
			self.log.error(str(ex))
			from traceback import format_exc
			self.log.debug(format_exc())
			return -1

		return 0
コード例 #7
0
ファイル: server.py プロジェクト: philloidin/wok
    def _init_conf(self, app, conf_files, path_var, files_var, args_var):
        self.logger.info("Checking Wok configuration files ...")

        args = []

        conf_path = app.config.get(path_var,
                                   os.environ.get(path_var, os.getcwd()))
        wok_conf_files = _get_conf_files(
            conf_path, conf_files or [],
            app.config.get(files_var, os.environ.get(files_var)), ".conf")
        for path in wok_conf_files:
            if not os.path.exists(path):
                self.logger.error("--- {} (not found)".format(path))
                continue
            self.logger.info("+++ {}".format(path))
            args += ["-c", path]

        env_args = os.environ.get(args_var)
        if env_args is not None:
            env_args = env_args.strip().split(" ")

        wok_conf_args = app.config.get(args_var, env_args)
        if wok_conf_args is not None:
            args += wok_conf_args
            self.logger.debug("Arguments: {}".format(" ".join(wok_conf_args)))

        self.logger.info("Loading Wok configuration ...")

        parser = ArgumentParser()
        wok_args = Arguments(parser, case_name_args=False, logger=self.logger)
        wok_args.initialize(parser.parse_args(args))

        self.conf_builder = wok_args.engine_conf_builder
        self.conf = wok_args.engine_conf.expand_vars()

        self.case_conf_builder = wok_args.case_conf_builder
        self.case_conf = wok_args.case_conf.expand_vars()

        # initialize logging according to the configuration

        log = logger.get_logger("")
        log.removeHandler(log.handlers[0])
        logging_conf = self.conf.clone().expand_vars().get("wok.logging")
        logger.initialize(logging_conf)

        self.logger.debug(repr(self.conf))
コード例 #8
0
ファイル: server.py プロジェクト: chris-zen/phd-thesis
	def _init_conf(self, app, conf_files, path_var, files_var, args_var):
		self.logger.info("Checking Wok configuration files ...")

		args = []

		conf_path = app.config.get(path_var, os.environ.get(path_var, os.getcwd()))
		wok_conf_files = _get_conf_files(conf_path, conf_files or [],
										 app.config.get(files_var, os.environ.get(files_var)), ".conf")
		for path in wok_conf_files:
			if not os.path.exists(path):
				self.logger.error("--- {} (not found)".format(path))
				continue
			self.logger.info("+++ {}".format(path))
			args += ["-c", path]

		env_args = os.environ.get(args_var)
		if env_args is not None:
			env_args = env_args.strip().split(" ")

		wok_conf_args = app.config.get(args_var, env_args)
		if wok_conf_args is not None:
			args += wok_conf_args
			self.logger.debug("Arguments: {}".format(" ".join(wok_conf_args)))

		self.logger.info("Loading Wok configuration ...")

		parser = ArgumentParser()
		wok_args = Arguments(parser, case_name_args=False, logger=self.logger)
		wok_args.initialize(parser.parse_args(args))

		self.conf_builder = wok_args.engine_conf_builder
		self.conf = wok_args.engine_conf.expand_vars()

		self.case_conf_builder = wok_args.case_conf_builder
		self.case_conf = wok_args.case_conf.expand_vars()
		
		# initialize logging according to the configuration

		log = logger.get_logger("")
		log.removeHandler(log.handlers[0])
		logging_conf = self.conf.clone().expand_vars().get("wok.logging")
		logger.initialize(logging_conf)

		self.logger.debug(repr(self.conf))
コード例 #9
0
ファイル: task.py プロジェクト: philloidin/wok
    def __init__(self):

        # Get task key and storage configuration
        cmd_conf = OptionsConfig(
            required=["case", "task", "index", "data.type", "storage.type"])

        # Register signals
        self._signal_names = {}
        for signame in [x for x in dir(signal) if x.startswith("SIG")]:
            try:
                signum = getattr(signal, signame)
                signal.signal(signum, self.__signal_handler)
                self._signal_names[signum] = signame
            except:
                pass

        # command line configuration
        case_name = cmd_conf["case"]
        task_cname = cmd_conf["task"]
        workitem_index = cmd_conf["index"]

        # initialize the data provider
        provider_conf = cmd_conf["data"]
        self._provider = data_provider_factory.create(provider_conf)
        self._provider.start()

        # initialize storage
        storage_conf = cmd_conf["storage"]
        self.storage = storage_factory.create(storage_conf)
        self.storage = self.storage.get_container(case_name)

        # load the module and task descriptors
        task_desc = self._provider.load_task(case_name, task_cname)
        workitem_desc = self._provider.load_workitem(case_name, task_cname,
                                                     workitem_index)
        partition = workitem_desc["partition"]

        # setup task configuration
        self.conf = Data.create(task_desc["conf"])
        self.conf["__task_index"] = workitem_index
        self.conf.expand_vars()

        # setup task attributes
        self.case = workitem_desc["case"]
        self.task = workitem_desc["task"]
        self.id = workitem_desc["cname"]
        self.name = workitem_desc["name"]
        self.index = workitem_index

        # initialize decorators
        self._main = None
        self._sources = []
        self._foreach = None
        self._begin = None
        self._end = None

        self._start_time = 0
        self._end_time = self._start_time

        # intialize task logging
        log_conf = self.conf.get("logging")
        logger.initialize(log_conf)
        self.logger = logger.get_logger(self.name)

        self.logger.debug("Task descriptor: {}".format(Data.create(task_desc)))
        self.logger.debug("WorkItem descriptor: {}".format(
            Data.create(workitem_desc)))

        # Initialize input stream
        self._stream = Stream(self._provider, task_desc["stream"])

        # Initialize ports
        self._ports = {}
        self._in_ports = []
        self._out_ports = []

        self._open_ports = {}

        if "ports" in task_desc and "ports" in partition:
            port_descriptors = Data.create(task_desc["ports"])

            for port_desc in port_descriptors.get("in", default=list):
                port_desc["mode"] = PORT_MODE_IN
                self._ports[port_desc["name"]] = port_desc
                self._in_ports += [port_desc]

            for port_desc in port_descriptors.get("out", default=list):
                port_desc["mode"] = PORT_MODE_OUT
                self._ports[port_desc["name"]] = port_desc
                self._out_ports += [port_desc]

            port_descriptors = Data.create(partition["ports"])

            for port_desc in port_descriptors.get("in", default=list):
                task_port_desc = self._ports[port_desc["name"]]
                task_port_desc["data"] = port_desc["data"]

            for port_desc in port_descriptors.get("out", default=list):
                task_port_desc = self._ports[port_desc["name"]]
                task_port_desc["data"] = port_desc["data"]

        # Get hostname
        try:
            import socket
            self.hostname = socket.gethostname()
        except:
            self.hostname = "unknown"

        # The context field is free to be used by the task user to
        # save variables related with the whole task life cycle.
        # By default it is initialized with a dictionary but can be
        # overwrote with any value by the user. Wok will never use it.
        self.context = {}
コード例 #10
0
ファイル: task.py プロジェクト: bbglab/wok
	def __init__(self):

		# Get task key and storage configuration
		cmd_conf = OptionsConfig(required=["case", "task", "index", "data.type", "storage.type"])

		# Register signals
		self._signal_names = {}
		for signame in [x for x in dir(signal) if x.startswith("SIG")]:
			try:
				signum = getattr(signal, signame)
				signal.signal(signum, self.__signal_handler)
				self._signal_names[signum] = signame
			except:
				pass

		# command line configuration
		case_name = cmd_conf["case"]
		task_cname = cmd_conf["task"]
		workitem_index = cmd_conf["index"]

		# initialize the data provider
		provider_conf = cmd_conf["data"]
		self._provider = data_provider_factory.create(provider_conf)
		self._provider.start()

		# initialize storage
		storage_conf = cmd_conf["storage"]
		self.storage = storage_factory.create(storage_conf)
		self.storage = self.storage.get_container(case_name)

		# load the module and task descriptors
		task_desc = self._provider.load_task(case_name, task_cname)
		workitem_desc = self._provider.load_workitem(case_name, task_cname, workitem_index)
		partition = workitem_desc["partition"]

		# setup task configuration
		self.conf = Data.create(task_desc["conf"])
		self.conf["__task_index"] = workitem_index
		self.conf.expand_vars()

		# setup task attributes
		self.case = workitem_desc["case"]
		self.task = workitem_desc["task"]
		self.id = workitem_desc["cname"]
		self.name = workitem_desc["name"]
		self.index = workitem_index

		# initialize decorators
		self._main = None
		self._sources = []
		self._foreach = None
		self._begin = None
		self._end = None

		self._start_time = 0
		self._end_time = self._start_time

		# intialize task logging
		log_conf = self.conf.get("logging")
		logger.initialize(log_conf)
		self.logger = logger.get_logger(self.name)

		self.logger.debug("Task descriptor: {}".format(Data.create(task_desc)))
		self.logger.debug("WorkItem descriptor: {}".format(Data.create(workitem_desc)))

		# Initialize input stream
		self._stream = Stream(self._provider, task_desc["stream"])

		# Initialize ports
		self._ports = {}
		self._in_ports = []
		self._out_ports = []

		self._open_ports = {}

		if "ports" in task_desc and "ports" in partition:
			port_descriptors = Data.create(task_desc["ports"])

			for port_desc in port_descriptors.get("in", default=list):
				port_desc["mode"] = PORT_MODE_IN
				self._ports[port_desc["name"]] = port_desc
				self._in_ports += [port_desc]

			for port_desc in port_descriptors.get("out", default=list):
				port_desc["mode"] = PORT_MODE_OUT
				self._ports[port_desc["name"]] = port_desc
				self._out_ports += [port_desc]

			port_descriptors = Data.create(partition["ports"])

			for port_desc in port_descriptors.get("in", default=list):
				task_port_desc = self._ports[port_desc["name"]]
				task_port_desc["data"] = port_desc["data"]

			for port_desc in port_descriptors.get("out", default=list):
				task_port_desc = self._ports[port_desc["name"]]
				task_port_desc["data"] = port_desc["data"]

		# Get hostname
		try:
			import socket
			self.hostname = socket.gethostname()
		except:
			self.hostname = "unknown"

		# The context field is free to be used by the task user to
		# save variables related with the whole task life cycle.
		# By default it is initialized with a dictionary but can be
		# overwrote with any value by the user. Wok will never use it.
		self.context = {}
コード例 #11
0
ファイル: task.py プロジェクト: dalloliogm/wok
    def __init__(self):

        # Get task key and storage configuration
        cmd_conf = OptionsConfig(required=["instance_name", "module_path", "task_index", "storage.type"])

        instance_name = cmd_conf["instance_name"]
        module_path = cmd_conf["module_path"]
        task_index = cmd_conf["task_index"]

        storage_conf = cmd_conf["storage"]
        storage = create_storage(storage_conf["type"], StorageContext.EXECUTION, storage_conf)

        # Read data and configuration
        self.data = storage.load_task_config(instance_name, module_path, task_index)

        self.conf = self.data["conf"]
        del self.data["conf"]

        # set per task configuration values and expand
        self.conf["__task_index"] = task_index
        self.conf.expand_vars()

        self.id = self.data["id"]
        self.name = self.data["name"]
        self.module_path = self.data["module"]
        self.instance_name = self.data["instance"]

        self._main = None
        self._generators = []
        self._foreach = None
        self._begin = None
        self._end = None

        self._start_time = 0
        self._end_time = self._start_time

        logger.initialize(self.conf.get("log"))
        self._log = self.logger()

        # Initialize data iteration
        iter_conf = self.data["iteration"]
        self._iter_strategy = iter_conf.get("strategy")
        self._iter_size = iter_conf.get("size", 0, dtype=int)

        # Initialize ports
        self._port_map = {}
        self._in_ports = []
        self._out_ports = []

        if "ports" in self.data:
            ports_conf = self.data["ports"]

            for port_conf in ports_conf.get("in", []):
                port = create_port(PORT_MODE_IN, port_conf, storage)
                self._port_map[port.name] = port
                self._in_ports += [port]

            for port_conf in ports_conf.get("out", []):
                port = create_port(PORT_MODE_OUT, port_conf, storage)
                self._port_map[port.name] = port
                self._out_ports += [port]

        self.__ports_accessor = PortsAccessor(self._port_map)

        # The context field is free to be used by the task user to
        # save variables related with the whole task life cycle.
        # By default it is initialized with a dictionary but can be
        # overwrited with any value by the user. Wok will never use it.
        self.context = {}