def __init__(self, desc, name=None, logger=None, base_path=None): self._log = logger or woklogger.get_logger(_LOGGER_NAME) self.name = name or desc.get("name") self._flow_loader = None platform = desc.get("platform") self.path = desc.get("path") self.conf = desc.get("conf", default=Data.element) if isinstance(self.conf, basestring): self.conf = ConfigLoader(os.path.join(base_path or "", self.conf)).load() self.conf_rules = [] if self.path is not None: self.conf_rules += [ ConfRule(dict(set=[[rtconf.PROJECT_PATH, self.path]]), base_path, platform) ] self.conf_rules += [ ConfRule(rule, base_path, platform) for rule in desc.get("conf_rules", default=Data.list) ] self.flows = desc.get("flows", default=Data.list)
def _create_platforms(self): """ Creates the platform according to the configuration :return: Platform """ platform_confs = self._conf.get("platforms") if platform_confs is None: platform_confs = Data.list() elif not Data.is_list(platform_confs): self._log.error("Wrong configuration type for 'platforms': {}".format(platform_confs)) platform_confs = Data.list() if len(platform_confs) == 0: platform_confs += [Data.element(dict(type="local"))] platforms = [] names = {} for pidx, platform_conf in enumerate(platform_confs): if isinstance(platform_conf, basestring): if not os.path.isabs(platform_conf) and self._conf_base_path is not None: platform_conf = os.path.join(self._conf_base_path, platform_conf) platform_conf = ConfigLoader(platform_conf).load() if not Data.is_element(platform_conf): raise errors.ConfigTypeError("wok.platforms[{}]".format(pidx, platform_conf)) ptype = platform_conf.get("type", "local") name = platform_conf.get("name", ptype) if name in names: name = "{}-{}".format(name, names[name]) names[name] += 1 else: names[name] = 2 platform_conf["name"] = name if "work_path" not in platform_conf: platform_conf["work_path"] = os.path.join(self._work_path, "platform_{}".format(name)) self._log.info("Creating '{}' platform ...".format(name)) self._log.debug("Platform configuration: {}".format(repr(platform_conf))) platforms += [create_platform(ptype, platform_conf)] return platforms
def _load_project_desc(self, path, base_path=None): if not os.path.isabs(path): if base_path is not None: path = os.path.join(base_path, path) else: path = os.path.abspath(path) if not os.path.exists(path): raise Exception("Project path not found: {}".format(path)) if os.path.isdir(path): path = os.path.join(path, "project.conf") if not os.path.isfile(path): raise Exception("Project configuration not found: {}".format(path)) project = Data.element() project.merge(ConfigLoader(path).load()) base_path = os.path.dirname(path) if "path" not in project: project["path"] = base_path if not os.path.isabs(project["path"]): project["path"] = os.path.normpath( os.path.join(base_path, project["path"])) if "conf" in project and isinstance(project["conf"], basestring): conf_path = os.path.join(base_path, project["conf"]) project["conf"] = ConfigLoader(conf_path).load() if "conf_rules" in project and isinstance(project["conf_rules"], basestring): base_path = os.path.dirname(path) conf_path = os.path.join(base_path, project["conf_rules"]) project["conf_rules"] = ConfigLoader(conf_path).load() if "conf_rules" in project and Data.is_list(project["conf_rules"]): for rule in project["conf_rules"]: if Data.is_element(rule) and "merge" in rule and isinstance( rule["merge"], basestring): rule["merge"] = ConfigLoader( os.path.join(base_path, rule["merge"])).load() return project
def __init__(self, rule, base_path=None, platform=None): rule = Data.create(rule) self.on = rule.get("on", {}) if isinstance(self.on, basestring): self.on = dict(task=self.on) if platform is not None: self.on["platform"] = platform self.dels = rule.get("del", default=Data.list) if not Data.is_list(self.dels): raise Exception( "Expected a list of strings for del operations of rule: {}". format(repr(rule))) for k in self.dels: if not isinstance(k, basestring): raise Exception( "Expected a list of strings for del operations of rule: {}" .format(repr(rule))) self.set = rule.get("set", default=Data.list) if not Data.is_list(self.dels): raise Exception( "Expected a list of tuples [key, value] for set operations of rule: {}" .format(repr(rule))) for s in self.set: if not Data.is_list(s) or len(s) != 2: raise Exception( "Expected a list of tuples [key, value] for set operations of rule: {}" .format(repr(rule))) self.merge = rule.get("merge") if isinstance(self.merge, basestring): if not os.path.isabs(self.merge): if base_path is None: raise Exception( "Configuration rule merge path should be absolute path: {}" .format(self.merge)) else: self.merge = os.path.join(base_path, self.merge) if not os.path.isfile(self.merge): raise Exception( "Configuration rule merge path not found: {}".format( self.merge)) self.merge = ConfigLoader(os.path.join(base_path or "", self.merge)).load() if self.merge is not None and not Data.is_element(self.merge): raise Exception( "Expected a dictionary for merge operation of rule: {}".format( repr(rule)))
def __init__(self, desc, name=None, logger=None, base_path=None): self._log = logger or woklogger.get_logger(_LOGGER_NAME) self.name = name or desc.get("name") self._flow_loader = None platform = desc.get("platform") self.path = desc.get("path") self.conf = desc.get("conf", default=Data.element) if isinstance(self.conf, basestring): self.conf = ConfigLoader(os.path.join(base_path or "", self.conf)).load() self.conf_rules = [] if self.path is not None: self.conf_rules += [ConfRule(dict(set=[[rtconf.PROJECT_PATH, self.path]]), base_path, platform)] self.conf_rules += [ConfRule(rule, base_path, platform) for rule in desc.get("conf_rules", default=Data.list)] self.flows = desc.get("flows", default=Data.list)
class Project(object): def __init__(self, desc, name=None, logger=None, base_path=None): self._log = logger or woklogger.get_logger(_LOGGER_NAME) self.name = name or desc.get("name") self._flow_loader = None platform = desc.get("platform") self.path = desc.get("path") self.conf = desc.get("conf", default=Data.element) if isinstance(self.conf, basestring): self.conf = ConfigLoader(os.path.join(base_path or "", self.conf)).load() self.conf_rules = [] if self.path is not None: self.conf_rules += [ConfRule(dict(set=[[rtconf.PROJECT_PATH, self.path]]), base_path, platform)] self.conf_rules += [ConfRule(rule, base_path, platform) for rule in desc.get("conf_rules", default=Data.list)] self.flows = desc.get("flows", default=Data.list) def extend(self, pdesc, base_path=None): platform = pdesc.get("platform") if platform is not None and "path" in pdesc: self.conf_rules += [ConfRule(dict(set=[[rtconf.PROJECT_PATH, pdesc["path"]]]), base_path, platform)] if "conf" in pdesc: if platform is None: self.conf.merge(pdesc["conf"]) else: self.conf_rules += [ConfRule(dict(merge=pdesc["conf"]), base_path, platform)] self.conf_rules += [ConfRule(rule, base_path, platform) for rule in pdesc.get("conf_rules", default=Data.list)] self.flows += pdesc.get("flows", default=Data.list) def initialize(self): self._log.info("Initializing project {} ...".format(self.name)) if self.path is None: raise Exception("Project 'path' not defined: {}".format(self.name)) flow_paths = [os.path.join(self.path, flow) for flow in self.flows] self._flow_loader = FlowLoader(flow_paths) for uri, path in self._flow_loader.flow_files.items(): self._log.debug("{0} : {1}".format(uri, path)) def load_flow(self, flow_name, version=None): flow = self._flow_loader.load_from_canonical(flow_name, version=version) flow.project = self return flow def load_from_ref(self, ref): flow = self._flow_loader.load_from_ref(ref) flow.project = self return flow def get_conf(self, task_name=None, platform_name=None): conf = self.conf.clone() for rule in self.conf_rules: pname = conf.get(rtconf.PLATFORM_TARGET, platform_name) if rule.match(task_name=task_name, platform=pname): rule.apply(conf) if rtconf.PROJECT_PATH not in conf: conf[rtconf.PROJECT_PATH] = self.path return conf def _repr(self, sb, indent=0): sb += ["Project [{}]".format(self.name)] sb += [" path={}".format(self.path)] sb += [" flows={}".format(self.flows)] sb += [" conf={}".format(self.conf)] sb += [" conf_rules={}".format(self.conf_rules)] for platform_name in sorted(self._platforms.keys()): self._platforms[platform_name]._repr(sb, 2) def __repr__(self): sb = [] self._repr(sb) return "\n".join(sb)
class Project(object): def __init__(self, desc, name=None, logger=None, base_path=None): self._log = logger or woklogger.get_logger(_LOGGER_NAME) self.name = name or desc.get("name") self._flow_loader = None platform = desc.get("platform") self.path = desc.get("path") self.conf = desc.get("conf", default=Data.element) if isinstance(self.conf, basestring): self.conf = ConfigLoader(os.path.join(base_path or "", self.conf)).load() self.conf_rules = [] if self.path is not None: self.conf_rules += [ ConfRule(dict(set=[[rtconf.PROJECT_PATH, self.path]]), base_path, platform) ] self.conf_rules += [ ConfRule(rule, base_path, platform) for rule in desc.get("conf_rules", default=Data.list) ] self.flows = desc.get("flows", default=Data.list) def extend(self, pdesc, base_path=None): platform = pdesc.get("platform") if platform is not None and "path" in pdesc: self.conf_rules += [ ConfRule(dict(set=[[rtconf.PROJECT_PATH, pdesc["path"]]]), base_path, platform) ] if "conf" in pdesc: if platform is None: self.conf.merge(pdesc["conf"]) else: self.conf_rules += [ ConfRule(dict(merge=pdesc["conf"]), base_path, platform) ] self.conf_rules += [ ConfRule(rule, base_path, platform) for rule in pdesc.get("conf_rules", default=Data.list) ] self.flows += pdesc.get("flows", default=Data.list) def initialize(self): self._log.info("Initializing project {} ...".format(self.name)) if self.path is None: raise Exception("Project 'path' not defined: {}".format(self.name)) flow_paths = [os.path.join(self.path, flow) for flow in self.flows] self._flow_loader = FlowLoader(flow_paths) for uri, path in self._flow_loader.flow_files.items(): self._log.debug("{0} : {1}".format(uri, path)) def load_flow(self, flow_name, version=None): flow = self._flow_loader.load_from_canonical(flow_name, version=version) flow.project = self return flow def load_from_ref(self, ref): flow = self._flow_loader.load_from_ref(ref) flow.project = self return flow def get_conf(self, task_name=None, platform_name=None): conf = self.conf.clone() for rule in self.conf_rules: pname = conf.get(rtconf.PLATFORM_TARGET, platform_name) if rule.match(task_name=task_name, platform=pname): rule.apply(conf) if rtconf.PROJECT_PATH not in conf: conf[rtconf.PROJECT_PATH] = self.path return conf def _repr(self, sb, indent=0): sb += ["Project [{}]".format(self.name)] sb += [" path={}".format(self.path)] sb += [" flows={}".format(self.flows)] sb += [" conf={}".format(self.conf)] sb += [" conf_rules={}".format(self.conf_rules)] for platform_name in sorted(self._platforms.keys()): self._platforms[platform_name]._repr(sb, 2) def __repr__(self): sb = [] self._repr(sb) return "\n".join(sb)