def __init__(self, rule, base_path=None, platform=None): rule = Data.create(rule) self.on = rule.get("on", {}) if isinstance(self.on, basestring): self.on = dict(task=self.on) if platform is not None: self.on["platform"] = platform self.dels = rule.get("del", default=Data.list) if not Data.is_list(self.dels): raise Exception( "Expected a list of strings for del operations of rule: {}". format(repr(rule))) for k in self.dels: if not isinstance(k, basestring): raise Exception( "Expected a list of strings for del operations of rule: {}" .format(repr(rule))) self.set = rule.get("set", default=Data.list) if not Data.is_list(self.dels): raise Exception( "Expected a list of tuples [key, value] for set operations of rule: {}" .format(repr(rule))) for s in self.set: if not Data.is_list(s) or len(s) != 2: raise Exception( "Expected a list of tuples [key, value] for set operations of rule: {}" .format(repr(rule))) self.merge = rule.get("merge") if isinstance(self.merge, basestring): if not os.path.isabs(self.merge): if base_path is None: raise Exception( "Configuration rule merge path should be absolute path: {}" .format(self.merge)) else: self.merge = os.path.join(base_path, self.merge) if not os.path.isfile(self.merge): raise Exception( "Configuration rule merge path not found: {}".format( self.merge)) self.merge = ConfigLoader(os.path.join(base_path or "", self.merge)).load() if self.merge is not None and not Data.is_element(self.merge): raise Exception( "Expected a dictionary for merge operation of rule: {}".format( repr(rule)))
def single_project(projects_port): log = task.logger conf = task.conf log.info("Single project ...") files = conf["files"] if isinstance(files, basestring): files = [files] elif not Data.is_list(files): raise InternalError("Unexpected configuration type for 'files', only string and list are allowed") files = [str(f) for f in files] #unicode is not json serializable project_id = conf["project.id"] assembly = conf.get("assembly", "hg19") annotations = conf.get("annotations") project = { "id" : project_id, "files" : files, "assembly" : assembly } if annotations is not None and isinstance(annotations, dict): project["annotations"] = annotations make_project(log, conf, project) projects_port.send(project)
def get_handler(logger, conf): if not Data.is_element(conf): logger.error("Malformed log handler:\n{1}".format(repr(conf))) return type = conf.get("type") if type is None or not isinstance( type, basestring) or type.lower() not in _HANDLERS.keys(): logger.error("Unknown or unsupported handler type: {0}\n{1}".format( type, repr(conf))) return handler = _HANDLERS[type](conf) level = conf.get("level") if level is not None: handler.setLevel(get_level(level)) format = conf.get("format", _DEFAULT_FORMAT) if format is not None: if Data.is_list(format): format = "".join(format.to_native()) handler.setFormatter(logging.Formatter(format)) return handler
def get_annotations_to_save(self, keys, annotations, names=None, values=None): assert (names is None and values is None) or (names is not None and values is not None and len(names) == len(values)) if names is None: names = [] values = [] else: names = [name for name in names] values = [value for value in values] ann_keys = keys if ann_keys is None: ann_keys = [] elif Data.is_list(ann_keys): ann_keys = ann_keys.to_native() else: log = logger.get_logger(__name__) log.warn("Wrong type for 'project.annotations', expecting a list but found:\n{0}".format(repr(ann_keys))) ann_keys = [] for ann_key in ann_keys: default = None if isinstance(ann_key, basestring): key = name = ann_key elif isinstance(ann_key, list) and len(ann_key) == 2: key = ann_key[0] name = ann_key[1] value = annotations[key] if key in annotations else default names += [name] values += [value] return names, values
def __init__(self, conf): JobManager.__init__(self, "saga", conf) self._file_url = self._conf.get("files_url", "file://") self._remote_path = self._conf.get("remote_path", self._work_path) self._output_path = os.path.join(self._work_path, "output") self._remote_output_path = os.path.join(self._remote_path, "output") self._pe = self._conf.get("pe") self._cpu_count = self._conf.get("cpu_count", 1) self._queue = self._conf.get("queue") self._project = self._conf.get("project") self._working_directory = self._conf.get("working_directory") self._state_check_interval = self._conf.get("state_check_interval", 5) ctx_conf = self._conf.get("context") if ctx_conf is not None and not (Data.is_element(ctx_conf) or Data.is_list(ctx_conf)): raise ConfigTypeError("context", ctx_conf) self._session = None self._job_service = None self._queued_count = 0 self._max_queued = self._conf.get("max_queued", 0) self._running = False self._run_thread = None self._join_thread = None
def scan_projects(projects_port): log = task.logger conf = task.conf config = GlobalConfig(conf) log.debug("Configuration:") log.debug(config) log.info("Creating combination folders ...") paths = PathsConfig(config) create_combination_folders(conf) if "projects" not in conf or not Data.is_list(conf["projects"]): raise InternalError("Required 'projects' configuration parameter has not been found or it is not well defined") log.info("Initializing projects ...") for project in conf["projects"]: if project is None: continue project = validate_project(log, project) project.expand_vars(conf) log.info("--- [{}] ---------------------------------------------------".format(project["id"])) project = init_project(log, config, paths, task.storage, project) log.info(" assembly: {}, variant_files: {}".format(project["assembly"], len(project["files"]))) projects_port.send(project)
def __init__(self, rule, base_path=None, platform=None): rule = Data.create(rule) self.on = rule.get("on", {}) if isinstance(self.on, basestring): self.on = dict(task=self.on) if platform is not None: self.on["platform"] = platform self.dels = rule.get("del", default=Data.list) if not Data.is_list(self.dels): raise Exception("Expected a list of strings for del operations of rule: {}".format(repr(rule))) for k in self.dels: if not isinstance(k, basestring): raise Exception("Expected a list of strings for del operations of rule: {}".format(repr(rule))) self.set = rule.get("set", default=Data.list) if not Data.is_list(self.dels): raise Exception("Expected a list of tuples [key, value] for set operations of rule: {}".format(repr(rule))) for s in self.set: if not Data.is_list(s) or len(s) != 2: raise Exception("Expected a list of tuples [key, value] for set operations of rule: {}".format(repr(rule))) self.merge = rule.get("merge") if isinstance(self.merge, basestring): if not os.path.isabs(self.merge): if base_path is None: raise Exception("Configuration rule merge path should be absolute path: {}".format(self.merge)) else: self.merge = os.path.join(base_path, self.merge) if not os.path.isfile(self.merge): raise Exception("Configuration rule merge path not found: {}".format(self.merge)) self.merge = ConfigLoader(os.path.join(base_path or "", self.merge)).load() if self.merge is not None and not Data.is_element(self.merge): raise Exception("Expected a dictionary for merge operation of rule: {}".format(repr(rule)))
def _create_platforms(self): """ Creates the platform according to the configuration :return: Platform """ platform_confs = self._conf.get("platforms") if platform_confs is None: platform_confs = Data.list() elif not Data.is_list(platform_confs): self._log.error("Wrong configuration type for 'platforms': {}".format(platform_confs)) platform_confs = Data.list() if len(platform_confs) == 0: platform_confs += [Data.element(dict(type="local"))] platforms = [] names = {} for pidx, platform_conf in enumerate(platform_confs): if isinstance(platform_conf, basestring): if not os.path.isabs(platform_conf) and self._conf_base_path is not None: platform_conf = os.path.join(self._conf_base_path, platform_conf) platform_conf = ConfigLoader(platform_conf).load() if not Data.is_element(platform_conf): raise errors.ConfigTypeError("wok.platforms[{}]".format(pidx, platform_conf)) ptype = platform_conf.get("type", "local") name = platform_conf.get("name", ptype) if name in names: name = "{}-{}".format(name, names[name]) names[name] += 1 else: names[name] = 2 platform_conf["name"] = name if "work_path" not in platform_conf: platform_conf["work_path"] = os.path.join(self._work_path, "platform_{}".format(name)) self._log.info("Creating '{}' platform ...".format(name)) self._log.debug("Platform configuration: {}".format(repr(platform_conf))) platforms += [create_platform(ptype, platform_conf)] return platforms
def initialize(self): self._log.info("Initializing projects ...") if Data.is_element(self.conf): iter_conf = self._iter_dict(self.conf) elif Data.is_list(self.conf): iter_conf = self._iter_list(self.conf) else: iter_conf = iter([]) for name, pdesc in iter_conf: if isinstance(pdesc, basestring): pdesc = self._load_project_desc(pdesc, self.base_path) self._add_project_desc(pdesc, self.base_path) for name, project in sorted(self._projects.items(), key=lambda x: x[0]): project.initialize()
def _load_project_desc(self, path, base_path=None): if not os.path.isabs(path): if base_path is not None: path = os.path.join(base_path, path) else: path = os.path.abspath(path) if not os.path.exists(path): raise Exception("Project path not found: {}".format(path)) if os.path.isdir(path): path = os.path.join(path, "project.conf") if not os.path.isfile(path): raise Exception("Project configuration not found: {}".format(path)) project = Data.element() project.merge(ConfigLoader(path).load()) base_path = os.path.dirname(path) if "path" not in project: project["path"] = base_path if not os.path.isabs(project["path"]): project["path"] = os.path.normpath( os.path.join(base_path, project["path"])) if "conf" in project and isinstance(project["conf"], basestring): conf_path = os.path.join(base_path, project["conf"]) project["conf"] = ConfigLoader(conf_path).load() if "conf_rules" in project and isinstance(project["conf_rules"], basestring): base_path = os.path.dirname(path) conf_path = os.path.join(base_path, project["conf_rules"]) project["conf_rules"] = ConfigLoader(conf_path).load() if "conf_rules" in project and Data.is_list(project["conf_rules"]): for rule in project["conf_rules"]: if Data.is_element(rule) and "merge" in rule and isinstance( rule["merge"], basestring): rule["merge"] = ConfigLoader( os.path.join(base_path, rule["merge"])).load() return project
def _load_project_desc(self, path, base_path=None): if not os.path.isabs(path): if base_path is not None: path = os.path.join(base_path, path) else: path = os.path.abspath(path) if not os.path.exists(path): raise Exception("Project path not found: {}".format(path)) if os.path.isdir(path): path = os.path.join(path, "project.conf") if not os.path.isfile(path): raise Exception("Project configuration not found: {}".format(path)) project = Data.element() project.merge(ConfigLoader(path).load()) base_path = os.path.dirname(path) if "path" not in project: project["path"] = base_path if not os.path.isabs(project["path"]): project["path"] = os.path.normpath(os.path.join(base_path, project["path"])) if "conf" in project and isinstance(project["conf"], basestring): conf_path = os.path.join(base_path, project["conf"]) project["conf"] = ConfigLoader(conf_path).load() if "conf_rules" in project and isinstance(project["conf_rules"], basestring): base_path = os.path.dirname(path) conf_path = os.path.join(base_path, project["conf_rules"]) project["conf_rules"] = ConfigLoader(conf_path).load() if "conf_rules" in project and Data.is_list(project["conf_rules"]): for rule in project["conf_rules"]: if Data.is_element(rule) and "merge" in rule and isinstance(rule["merge"], basestring): rule["merge"] = ConfigLoader(os.path.join(base_path, rule["merge"])).load() return project
def initialize(conf=None, format=None, datefmt=None, level=None): """ Initialize the logging system. If conf is a dictionary then the parameters considered for configuration are: - format: Logger format - datefmt: Date format - loggers: list of tuples (name, conf) to configure loggers If conf is a list then only the loggers are configured. If conf is an string then the default logger is configured for the logging level. """ global _initialized if conf is None: conf = Data.element() elif not isinstance(conf, basestring): conf = Data.create(conf) if Data.is_list(conf): loggers_conf = conf conf = Data.element() elif Data.is_element(conf): loggers_conf = conf.get("loggers", default=Data.list) elif isinstance(conf, basestring): loggers_conf = Data.list([["", conf]]) conf = Data.element() format = format or conf.get("format", _DEFAULT_FORMAT) datefmt = datefmt or conf.get("datefmt", _DEFAULT_DATEFMT) logging.basicConfig(format=format, datefmt=datefmt) for (log_name, log_conf) in loggers_conf: init_logger(log_name, conf=log_conf) if level is not None: init_logger("", conf=level) _initialized = True
def get_handler(logger, conf): if not Data.is_element(conf): logger.error("Malformed log handler:\n{1}".format(repr(conf))) return type = conf.get("type") if type is None or not isinstance(type, basestring) or type.lower() not in _HANDLERS.keys(): logger.error("Unknown or unsupported handler type: {0}\n{1}".format(type, repr(conf))) return handler = _HANDLERS[type](conf) level = conf.get("level") if level is not None: handler.setLevel(get_level(level)) format = conf.get("format", _DEFAULT_FORMAT) if format is not None: if Data.is_list(format): format = "".join(format.to_native()) handler.setFormatter(logging.Formatter(format)) return handler
class NativeCommmandBuilder(CommmandBuilder): def _plain_conf(self, value, path=None): if path is None: path = [] if not Data.is_element(value): yield (".".join(path), value) else: for key in value.keys(): for k, v in self._plain_conf(value[key], path + [key]): yield (k, v) def prepare(self, case, task, index): execution = task.execution exec_conf = execution.conf if exec_conf is None: exec_conf = Data.element() if "script_path" not in exec_conf: raise MissingValueError("script_path") script_path = exec_conf["script_path"] lang = exec_conf.get("language", "python") case_conf = case.conf.clone().expand_vars() # Environment variables env = Data.element() #for k, v in os.environ.items(): # env[k] = v env.merge(task.conf.get(rtconf.TASK_ENV)) env.merge(exec_conf.get("env")) # Default module script path platform_project_path = task.conf.get(rtconf.PROJECT_PATH, case.project.path) flow_path = os.path.abspath(os.path.dirname(task.flow_path)) flow_rel_path = os.path.relpath(flow_path, case.project.path) platform_script_path = os.path.join(platform_project_path, flow_rel_path, script_path) env[ENV_PROJECT_PATH] = platform_project_path env[ENV_FLOW_PATH] = flow_rel_path env[ENV_SCRIPT_PATH] = script_path env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path script = [] sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list) if isinstance(sources, basestring): sources = Data.list([sources]) for source in sources: script += ['source "{}"'.format(source)] if lang == "python": virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV) if virtualenv is not None: #script += ["set -x"] #script += ["echo Activating virtualenv {} ...".format(virtualenv)] script += [ 'source "{}"'.format( os.path.join(virtualenv, "bin", "activate")) ] #script += ["set +x"] #script += ["echo Running workitem ..."] cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")] cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)] lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS) if lib_path is not None: if Data.is_list(lib_path): lib_path = ":".join(lib_path) if "PYTHONPATH" in env: env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"] else: env["PYTHONPATH"] = lib_path else: raise LanguageError(lang) cmd += [ "-D", "case={}".format(case.name), "-D", "task={}".format(task.cname), "-D", "index={}".format(index) ] #for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf): # cmd += ["-D", "storage.{}={}".format(key, value)] for key, value in self._plain_conf( Data.create(task.platform.data.context_conf(CTX_EXEC))): cmd += ["-D", "data.{}={}".format(key, value)] for key, value in self._plain_conf( task.platform.storage.context_conf(CTX_EXEC)): cmd += ["-D", "storage.{}={}".format(key, value)] script += [" ".join(cmd)] return "\n".join(script), env.to_native()
def prepare(self, case, task, index): execution = task.execution exec_conf = execution.conf if exec_conf is None: exec_conf = Data.element() if "script_path" not in exec_conf: raise MissingValueError("script_path") script_path = exec_conf["script_path"] lang = exec_conf.get("language", "python") case_conf = case.conf.clone().expand_vars() # Environment variables env = Data.element() #for k, v in os.environ.items(): # env[k] = v env.merge(task.conf.get(rtconf.TASK_ENV)) env.merge(exec_conf.get("env")) # Default module script path platform_project_path = task.conf.get(rtconf.PROJECT_PATH, case.project.path) flow_path = os.path.abspath(os.path.dirname(task.flow_path)) flow_rel_path = os.path.relpath(flow_path, case.project.path) platform_script_path = os.path.join(platform_project_path, flow_rel_path, script_path) env[ENV_PROJECT_PATH] = platform_project_path env[ENV_FLOW_PATH] = flow_rel_path env[ENV_SCRIPT_PATH] = script_path env[ENV_PLATFORM_SCRIPT_PATH] = platform_script_path script = [] sources = task.conf.get(rtconf.TASK_SOURCES, default=Data.list) if isinstance(sources, basestring): sources = Data.list([sources]) for source in sources: script += ['source "{}"'.format(source)] if lang == "python": virtualenv = task.conf.get(rtconf.TASK_PYTHON_VIRTUALENV) if virtualenv is not None: #script += ["set -x"] #script += ["echo Activating virtualenv {} ...".format(virtualenv)] script += ['source "{}"'.format(os.path.join(virtualenv, "bin", "activate"))] #script += ["set +x"] #script += ["echo Running workitem ..."] cmd = [task.conf.get(rtconf.TASK_PYTHON_BIN, "python")] cmd += ["${}".format(ENV_PLATFORM_SCRIPT_PATH)] lib_path = task.conf.get(rtconf.TASK_PYTHON_LIBS) if lib_path is not None: if Data.is_list(lib_path): lib_path = ":".join(lib_path) if "PYTHONPATH" in env: env["PYTHONPATH"] = lib_path + ":" + env["PYTHONPATH"] else: env["PYTHONPATH"] = lib_path else: raise LanguageError(lang) cmd += ["-D", "case={}".format(case.name), "-D", "task={}".format(task.cname), "-D", "index={}".format(index)] #for key, value in self._storage_conf(workitem.case.engine.storage.basic_conf): # cmd += ["-D", "storage.{}={}".format(key, value)] for key, value in self._plain_conf(Data.create(task.platform.data.context_conf(CTX_EXEC))): cmd += ["-D", "data.{}={}".format(key, value)] for key, value in self._plain_conf(task.platform.storage.context_conf(CTX_EXEC)): cmd += ["-D", "storage.{}={}".format(key, value)] script += [" ".join(cmd)] return "\n".join(script), env.to_native()
def multiple_projects(projects_port): log = task.logger conf = task.conf log.info("Scanning multiple projects ...") scan_paths = conf.get("scan_paths") if scan_paths is None or not Data.is_list(scan_paths): raise InternalError("Unexpected scan_paths type: {0}".format(type(scan_paths))) # Prepare includes and excludes includes = conf.get("includes") if includes is not None and not Data.is_list(includes): raise InternalError("Unexpected includes type: {0}".format(type(includes))) if includes is None or len(includes) == 0: includes = ["^.*$"] excludes = conf.get("excludes") if excludes is not None and not Data.is_list(excludes): raise InternalError("Unexpected excludes type: {0}".format(type(excludes))) if excludes is None: excludes = [] log.debug("scan_paths:\n{0}".format("\n ".join(conf["scan_paths"]))) log.debug("Includes:\n{0}".format("\n ".join(includes))) log.debug("Excludes:\n{0}".format("\n ".join(excludes))) # compile regular expressions includes = [re.compile(inc) for inc in includes] excludes = [re.compile(exc) for exc in excludes] # scan paths project_ids = set() for scan_path in scan_paths: for path, project in list_projects(log, scan_path): if "id" not in project: log.warn("Discarding project that doesn't have 'id': {0}".format(path)) continue if "files" not in project: log.warn("Discarding project that doesn't have 'files': {0}".format(path)) continue project_id = project["id"] if "name" in project: project_name = ": " + project["name"] else: project_name = "" if match_id(project_id, includes) and not match_id(project_id, excludes): if project_id in project_ids: raise Exception("Duplicated project id at {0}".format(path)) log.info("Included {0}{1}".format(project_id, project_name)) make_project(log, conf, project, base_path=os.path.dirname(path)) projects_port.send(project) else: log.info("Excluded {0}{1}".format(project_id, project_name))