def get_client(self, name: str, clients: List[Client], client_module: str) -> Union[ExecutionClient, InvalidClient]:
cl = [c for c in clients if c.name() == name]
if len(cl) > 0:
client = cl[0]
try:
mod = importlib.import_module(f'{client_module}.{name}.execution')
class_name = to_class_case(name + "_execution_client")
ms_client = getattr(mod, class_name)(client)
return ms_client
except Exception as e:
return InvalidClient(f"Could not instantiate execution client for: {name}: {message(e)}")
else:
return InvalidClient(f"Client not configured: {name}")
def build(self):
attributes = self.dict.attributes()
id = attributes.get("id")
mc = attributes.get("metastore_clients")
sc = attributes.get("storage_clients")
cc = attributes.get("scheduler_clients")
ec = attributes.get("execution_clients")
source_path = attributes.get("source")
cl = attributes.get("clients")
clients: List[Client] = []
invalid: List[InvalidClient] = []
if id:
# TODO: Add nested object and union support to typistry
# TODO: Clean this up
if isinstance(cl, dict):
for client_name, configuration in cl.items():
configuration = configuration.get("configuration")
if isinstance(configuration, dict):
configuration = safe_interpolate_environment(configuration)
client_class = self.supported_client(client_name)
if not client_class is None:
tdict = TypedDict(configuration, client_name)
valid: Union[ValidDict, InvalidObject] = validate_dict(tdict, from_root(self.client_path()))._inner_value
if isinstance(valid, ValidDict):
valid.typed_dict.type = valid.type() + "_client"
value: Union[Client, InvalidObject] = build_object(valid, to_class=client_class)._inner_value
if isinstance(value, InvalidObject):
invalid.append(InvalidClient(f"Invalid Client: {value.message}, {value.reference}"))
else:
clients.append(value)
else:
invalid.append(InvalidClient(f"{valid.message}"))
else:
invalid.append(InvalidClient(f"Client not supported: {client_name}"))
else:
invalid.append(InvalidClient(f"Bad Configuration. Must be dict: {configuration}"))
else:
invalid.append(InvalidClient("Bad client configuration"))
metastore_clients: List[Union[MetastoreClient, InvalidClient]] = MetastoreEngine().get_clients(mc, clients, self.client_module())
execution_clients: List[Union[ExecutionClient, InvalidClient]] = ExecutionEngine().get_clients(ec, clients, self.client_module())
scheduler_clients: List[Union[SchedulerClient, InvalidClient]] = SchedulerEngine().get_clients(cc, clients, self.client_module())
storage_clients: List[Union[StorageClient, InvalidClient]] = StorageEngine().get_clients(sc, clients, self.client_module())
return Config(id, clients, invalid, metastore_clients, execution_clients, storage_clients, scheduler_clients, source_path)
else:
return InvalidObject("Id not provided for config object", attributes)