def __init__(self, desc, name=None, logger=None, base_path=None):
self._log = logger or woklogger.get_logger(_LOGGER_NAME)
self.name = name or desc.get("name")
self._flow_loader = None
platform = desc.get("platform")
self.path = desc.get("path")
self.conf = desc.get("conf", default=Data.element)
if isinstance(self.conf, basestring):
self.conf = ConfigLoader(os.path.join(base_path or "",
self.conf)).load()
self.conf_rules = []
if self.path is not None:
self.conf_rules += [
ConfRule(dict(set=[[rtconf.PROJECT_PATH, self.path]]),
base_path, platform)
]
self.conf_rules += [
ConfRule(rule, base_path, platform)
for rule in desc.get("conf_rules", default=Data.list)
]
self.flows = desc.get("flows", default=Data.list)
def _create_platforms(self):
"""
Creates the platform according to the configuration
:return: Platform
"""
platform_confs = self._conf.get("platforms")
if platform_confs is None:
platform_confs = Data.list()
elif not Data.is_list(platform_confs):
self._log.error("Wrong configuration type for 'platforms': {}".format(platform_confs))
platform_confs = Data.list()
if len(platform_confs) == 0:
platform_confs += [Data.element(dict(type="local"))]
platforms = []
names = {}
for pidx, platform_conf in enumerate(platform_confs):
if isinstance(platform_conf, basestring):
if not os.path.isabs(platform_conf) and self._conf_base_path is not None:
platform_conf = os.path.join(self._conf_base_path, platform_conf)
platform_conf = ConfigLoader(platform_conf).load()
if not Data.is_element(platform_conf):
raise errors.ConfigTypeError("wok.platforms[{}]".format(pidx, platform_conf))
ptype = platform_conf.get("type", "local")
name = platform_conf.get("name", ptype)
if name in names:
name = "{}-{}".format(name, names[name])
names[name] += 1
else:
names[name] = 2
platform_conf["name"] = name
if "work_path" not in platform_conf:
platform_conf["work_path"] = os.path.join(self._work_path, "platform_{}".format(name))
self._log.info("Creating '{}' platform ...".format(name))
self._log.debug("Platform configuration: {}".format(repr(platform_conf)))
platforms += [create_platform(ptype, platform_conf)]
return platforms
def _load_project_desc(self, path, base_path=None):
if not os.path.isabs(path):
if base_path is not None:
path = os.path.join(base_path, path)
else:
path = os.path.abspath(path)
if not os.path.exists(path):
raise Exception("Project path not found: {}".format(path))
if os.path.isdir(path):
path = os.path.join(path, "project.conf")
if not os.path.isfile(path):
raise Exception("Project configuration not found: {}".format(path))
project = Data.element()
project.merge(ConfigLoader(path).load())
base_path = os.path.dirname(path)
if "path" not in project:
project["path"] = base_path
if not os.path.isabs(project["path"]):
project["path"] = os.path.normpath(
os.path.join(base_path, project["path"]))
if "conf" in project and isinstance(project["conf"], basestring):
conf_path = os.path.join(base_path, project["conf"])
project["conf"] = ConfigLoader(conf_path).load()
if "conf_rules" in project and isinstance(project["conf_rules"],
basestring):
base_path = os.path.dirname(path)
conf_path = os.path.join(base_path, project["conf_rules"])
project["conf_rules"] = ConfigLoader(conf_path).load()
if "conf_rules" in project and Data.is_list(project["conf_rules"]):
for rule in project["conf_rules"]:
if Data.is_element(rule) and "merge" in rule and isinstance(
rule["merge"], basestring):
rule["merge"] = ConfigLoader(
os.path.join(base_path, rule["merge"])).load()
return project
def __init__(self, rule, base_path=None, platform=None):
rule = Data.create(rule)
self.on = rule.get("on", {})
if isinstance(self.on, basestring):
self.on = dict(task=self.on)
if platform is not None:
self.on["platform"] = platform
self.dels = rule.get("del", default=Data.list)
if not Data.is_list(self.dels):
raise Exception(
"Expected a list of strings for del operations of rule: {}".
format(repr(rule)))
for k in self.dels:
if not isinstance(k, basestring):
raise Exception(
"Expected a list of strings for del operations of rule: {}"
.format(repr(rule)))
self.set = rule.get("set", default=Data.list)
if not Data.is_list(self.dels):
raise Exception(
"Expected a list of tuples [key, value] for set operations of rule: {}"
.format(repr(rule)))
for s in self.set:
if not Data.is_list(s) or len(s) != 2:
raise Exception(
"Expected a list of tuples [key, value] for set operations of rule: {}"
.format(repr(rule)))
self.merge = rule.get("merge")
if isinstance(self.merge, basestring):
if not os.path.isabs(self.merge):
if base_path is None:
raise Exception(
"Configuration rule merge path should be absolute path: {}"
.format(self.merge))
else:
self.merge = os.path.join(base_path, self.merge)
if not os.path.isfile(self.merge):
raise Exception(
"Configuration rule merge path not found: {}".format(
self.merge))
self.merge = ConfigLoader(os.path.join(base_path or "",
self.merge)).load()
if self.merge is not None and not Data.is_element(self.merge):
raise Exception(
"Expected a dictionary for merge operation of rule: {}".format(
repr(rule)))
def __init__(self, desc, name=None, logger=None, base_path=None):
self._log = logger or woklogger.get_logger(_LOGGER_NAME)
self.name = name or desc.get("name")
self._flow_loader = None
platform = desc.get("platform")
self.path = desc.get("path")
self.conf = desc.get("conf", default=Data.element)
if isinstance(self.conf, basestring):
self.conf = ConfigLoader(os.path.join(base_path or "", self.conf)).load()
self.conf_rules = []
if self.path is not None:
self.conf_rules += [ConfRule(dict(set=[[rtconf.PROJECT_PATH, self.path]]), base_path, platform)]
self.conf_rules += [ConfRule(rule, base_path, platform) for rule in desc.get("conf_rules", default=Data.list)]
self.flows = desc.get("flows", default=Data.list)
class Project(object):
def __init__(self, desc, name=None, logger=None, base_path=None):
self._log = logger or woklogger.get_logger(_LOGGER_NAME)
self.name = name or desc.get("name")
self._flow_loader = None
platform = desc.get("platform")
self.path = desc.get("path")
self.conf = desc.get("conf", default=Data.element)
if isinstance(self.conf, basestring):
self.conf = ConfigLoader(os.path.join(base_path or "", self.conf)).load()
self.conf_rules = []
if self.path is not None:
self.conf_rules += [ConfRule(dict(set=[[rtconf.PROJECT_PATH, self.path]]), base_path, platform)]
self.conf_rules += [ConfRule(rule, base_path, platform) for rule in desc.get("conf_rules", default=Data.list)]
self.flows = desc.get("flows", default=Data.list)
def extend(self, pdesc, base_path=None):
platform = pdesc.get("platform")
if platform is not None and "path" in pdesc:
self.conf_rules += [ConfRule(dict(set=[[rtconf.PROJECT_PATH, pdesc["path"]]]), base_path, platform)]
if "conf" in pdesc:
if platform is None:
self.conf.merge(pdesc["conf"])
else:
self.conf_rules += [ConfRule(dict(merge=pdesc["conf"]), base_path, platform)]
self.conf_rules += [ConfRule(rule, base_path, platform) for rule in pdesc.get("conf_rules", default=Data.list)]
self.flows += pdesc.get("flows", default=Data.list)
def initialize(self):
self._log.info("Initializing project {} ...".format(self.name))
if self.path is None:
raise Exception("Project 'path' not defined: {}".format(self.name))
flow_paths = [os.path.join(self.path, flow) for flow in self.flows]
self._flow_loader = FlowLoader(flow_paths)
for uri, path in self._flow_loader.flow_files.items():
self._log.debug("{0} : {1}".format(uri, path))
def load_flow(self, flow_name, version=None):
flow = self._flow_loader.load_from_canonical(flow_name, version=version)
flow.project = self
return flow
def load_from_ref(self, ref):
flow = self._flow_loader.load_from_ref(ref)
flow.project = self
return flow
def get_conf(self, task_name=None, platform_name=None):
conf = self.conf.clone()
for rule in self.conf_rules:
pname = conf.get(rtconf.PLATFORM_TARGET, platform_name)
if rule.match(task_name=task_name, platform=pname):
rule.apply(conf)
if rtconf.PROJECT_PATH not in conf:
conf[rtconf.PROJECT_PATH] = self.path
return conf
def _repr(self, sb, indent=0):
sb += ["Project [{}]".format(self.name)]
sb += [" path={}".format(self.path)]
sb += [" flows={}".format(self.flows)]
sb += [" conf={}".format(self.conf)]
sb += [" conf_rules={}".format(self.conf_rules)]
for platform_name in sorted(self._platforms.keys()):
self._platforms[platform_name]._repr(sb, 2)
def __repr__(self):
sb = []
self._repr(sb)
return "\n".join(sb)
class Project(object):
def __init__(self, desc, name=None, logger=None, base_path=None):
self._log = logger or woklogger.get_logger(_LOGGER_NAME)
self.name = name or desc.get("name")
self._flow_loader = None
platform = desc.get("platform")
self.path = desc.get("path")
self.conf = desc.get("conf", default=Data.element)
if isinstance(self.conf, basestring):
self.conf = ConfigLoader(os.path.join(base_path or "",
self.conf)).load()
self.conf_rules = []
if self.path is not None:
self.conf_rules += [
ConfRule(dict(set=[[rtconf.PROJECT_PATH, self.path]]),
base_path, platform)
]
self.conf_rules += [
ConfRule(rule, base_path, platform)
for rule in desc.get("conf_rules", default=Data.list)
]
self.flows = desc.get("flows", default=Data.list)
def extend(self, pdesc, base_path=None):
platform = pdesc.get("platform")
if platform is not None and "path" in pdesc:
self.conf_rules += [
ConfRule(dict(set=[[rtconf.PROJECT_PATH, pdesc["path"]]]),
base_path, platform)
]
if "conf" in pdesc:
if platform is None:
self.conf.merge(pdesc["conf"])
else:
self.conf_rules += [
ConfRule(dict(merge=pdesc["conf"]), base_path, platform)
]
self.conf_rules += [
ConfRule(rule, base_path, platform)
for rule in pdesc.get("conf_rules", default=Data.list)
]
self.flows += pdesc.get("flows", default=Data.list)
def initialize(self):
self._log.info("Initializing project {} ...".format(self.name))
if self.path is None:
raise Exception("Project 'path' not defined: {}".format(self.name))
flow_paths = [os.path.join(self.path, flow) for flow in self.flows]
self._flow_loader = FlowLoader(flow_paths)
for uri, path in self._flow_loader.flow_files.items():
self._log.debug("{0} : {1}".format(uri, path))
def load_flow(self, flow_name, version=None):
flow = self._flow_loader.load_from_canonical(flow_name,
version=version)
flow.project = self
return flow
def load_from_ref(self, ref):
flow = self._flow_loader.load_from_ref(ref)
flow.project = self
return flow
def get_conf(self, task_name=None, platform_name=None):
conf = self.conf.clone()
for rule in self.conf_rules:
pname = conf.get(rtconf.PLATFORM_TARGET, platform_name)
if rule.match(task_name=task_name, platform=pname):
rule.apply(conf)
if rtconf.PROJECT_PATH not in conf:
conf[rtconf.PROJECT_PATH] = self.path
return conf
def _repr(self, sb, indent=0):
sb += ["Project [{}]".format(self.name)]
sb += [" path={}".format(self.path)]
sb += [" flows={}".format(self.flows)]
sb += [" conf={}".format(self.conf)]
sb += [" conf_rules={}".format(self.conf_rules)]
for platform_name in sorted(self._platforms.keys()):
self._platforms[platform_name]._repr(sb, 2)
def __repr__(self):
sb = []
self._repr(sb)
return "\n".join(sb)
