class CfgLayer:
def __init__(self,
name: str,
include: list = None,
exclude: list = None,
file: str = None,
values: dict = None,
allow_unknown: bool = True):
self.name = name
self.file = file
self.values = from_file(file, {}, silent=True) if file else na(
values, {})
self.definitions = OrganizedList(__flags__.default_var_definitions(),
_key="name").filter(
None, include,
exclude) # type: OrganizedList
def update(self, key: str, value=None, delete: bool = False):
if not self.allow(key):
to_screen(f"{key} is not a recognized default variable, ignored")
return
dic = self.values
if delete:
if key not in dic:
to_screen(f"key {key} not found in {self.name}, ignored")
elif not self.act_append(
key) or not value: # delete the key when not append action
del dic[key]
to_screen(
f"key {key} removed completely from {self.name} successfully"
)
else:
dic[key].remove(value)
to_screen(
f"{value} removed in {key} under {self.name} successfully")
else:
if self.act_append(key):
def _append(dic, key, value):
dic.setdefault(key, [])
if value not in dic[key]:
dic[key].append(value)
_append(dic, key, value)
to_screen(
f"{value} added to {key} under {self.name} successfully")
else:
dic[key] = value
to_screen(
f"{key} set to {value} under {self.name} successfully")
if self.file:
to_file(self.values, self.file)
def allow(self, key: str):
return self.definitions.first_index(key) is not None
def act_append(self, key: str):
if self.allow(key):
return self.definitions.first(key).get("action", None) == "append"
return False
def __init__(self,
name: str,
include: list = None,
exclude: list = None,
file: str = None,
values: dict = None,
allow_unknown: bool = True):
self.name = name
self.file = file
self.values = from_file(file, {}, silent=True) if file else na(
values, {})
self.definitions = OrganizedList(__flags__.default_var_definitions(),
_key="name").filter(
None, include,
exclude) # type: OrganizedList
def reset(cls):
cls.definitions = OrganizedList(__flags__.default_var_definitions(),
_key="name").as_dict
cls.layers = OrganizedList([
CfgLayer(
name="user_advaced",
exclude=["clusters-in-local", "image-list", "resource-specs"]),
CfgLayer(
name="user_basic",
exclude=["clusters-in-local", "image-list", "resource-specs"]),
CfgLayer(name="local_default",
exclude=[],
file=__flags__.get_default_file(is_global=False)),
CfgLayer(name="global_default",
exclude=[],
file=__flags__.get_default_file(is_global=True))
],
_key="name",
_getter=getattr)
def select(self, alias: str=None):
"""return the cluster configuration (dict) with its alias equal to specified one
if only one cluster in the list and alias is not set, the only cluster would be returned"""
if not alias and len(self.clusters) == 1:
alias = self.clusters[0]["cluster_alias"]
__logger__.warn("cluster-alias is not set, the only defined cluster %s will be used", alias)
assert alias, "must specify a cluster_alias"
dic = ol.as_dict(self.clusters, "cluster_alias")
if alias not in dic:
os.system("cat %s" % __cluster_config_file__)
__logger__.error("cannot find %s from %s (%s)", alias, list(dic.keys()), __cluster_config_file__)
return dic[alias]
def test_dict(self):
lst = deepcopy(self.lst)
lst2 = deepcopy(lst)
# filter
self.assertDictEqual({
"matches": [lst[2]],
"indexes": [2]
}, ol.filter(lst, "a", "y"))
# as_dict
self.assertDictEqual(dict(x=lst[1], y=lst[2]), ol.as_dict(lst, "a"))
# add (update)
elem = dict(a="y", b="z", c="z")
lst2[2].update(elem)
self.assertTrue(ol.add(lst, "a", elem))
self.assertListEqual(lst, lst2)
# add append
elem = dict(a="z", f=4)
lst2.append(elem)
self.assertFalse(ol.add(lst, "a", elem))
self.assertListEqual(lst, lst2)
# delete
ol.delete(lst, "d", 2)
del lst2[2]
self.assertListEqual(lst, lst2)
def delete(self, alias: str):
return ol.delete(self.clusters, "cluster_alias", alias)
def add(self, cluster: dict):
return ol.notified_add(self.clusters, "cluster_alias",
Cluster.validate(Cluster.new(cluster)))
def attach_storage(c_dic: dict, storage: dict):
if "user" in storage and not storage["user"]:
storage["user"] = c_dic["user"]
return ol.notified_add(c_dic["storages"], "storage_alias", storage)
def add(self, cluster: dict):
cfg = Cluster().load(**cluster).check().config
ol.add(self.clusters, "cluster_alias", cfg)
return self
def do_action_list(self, args):
return ol.as_dict(self.__clusters__.tell(), "cluster_alias")
def do_action_list_storage(self, args):
return ol.as_dict(self.__clusters__.select(args.cluster_alias)['storages'], 'storage_alias')
def load(self, fname: str = None):
fname = na(fname, self.default_config_file)
self.clusters = OrganizedList(from_file(fname, default=[]),
_key="cluster_alias")
return self
def __init__(self, clusters: list = None):
self.clusters = OrganizedList(
clusters, _key="cluster_alias") if clusters else []
class ClusterList:
"""Data structure corresponding to the contents of ~/.openpai/clusters.yaml
We use an OrganizedList to handle the operations to this class
"""
def __init__(self, clusters: list = None):
self.clusters = OrganizedList(
clusters, _key="cluster_alias") if clusters else []
def load(self, fname: str = None):
fname = na(fname, self.default_config_file)
self.clusters = OrganizedList(from_file(fname, default=[]),
_key="cluster_alias")
return self
def save(self):
to_file(self.clusters.as_list, self.default_config_file)
@property
def default_config_file(self):
from openpaisdk.flags import __flags__
from openpaisdk.defaults import get_defaults
return __flags__.get_cluster_cfg_file(
get_defaults()["clusters-in-local"])
def tell(self):
return {
a: {
v: dict(GPUs='-',
memory='-',
vCores='-',
uri=cfg["pai_uri"],
user=cfg["user"])
for v in cfg["virtual_clusters"]
}
for a, cfg in self.clusters.as_dict.items()
}
def add(self, cluster: dict):
cfg = Cluster().load(**cluster).check().config
self.clusters.add(cfg, replace=True)
return self
def update_all(self):
for a in self.aliases:
self.add(self.clusters.first(a))
def delete(self, alias: str):
return self.clusters.remove(alias)
def select(self, alias: str):
return self.clusters.first(alias)
def get_client(self, alias: str):
return Cluster().load(**self.select(alias))
def available_resources(self):
"""concurrent version to get available resources"""
aliases = self.aliases
ret = concurrent_map(Cluster.available_resources,
(self.get_client(a) for a in aliases))
return {a: r for a, r in zip(aliases, ret) if r is not None}
@property
def aliases(self):
return [
c["cluster_alias"] for c in self.clusters if "cluster_alias" in c
]
@property
def alias(self):
return self.config["cluster_alias"]