def compile(self):
self._construct_train_dsl()
self._train_conf = self._construct_train_conf()
if self._stage == "predict":
predict_pipeline = self._predict_pipeline[0]["pipeline"]
data_info = self._predict_pipeline[0]["data"]
meta = predict_pipeline.get_predict_meta()
if meta["stage"] == "predict":
raise ValueError(
"adding predict pipeline objects'stage is predict, a predict pipeline cannot be an input component")
self._model_info = meta["model_info"]
predict_pipeline_dsl = meta["predict_dsl"]
predict_pipeline_conf = meta["train_conf"]
if not predict_pipeline_dsl:
raise ValueError(
"Cannot find deploy model in predict pipeline, to use a pipeline as input component, "
"it should be deploy first")
for cpn in self._train_dsl["components"]:
if cpn in predict_pipeline_dsl["components"]:
raise ValueError(
"component name {} exist in predict pipeline's deploy component, this is not support")
if "algorithm_parameters" in predict_pipeline_conf:
algo_param = predict_pipeline_conf["algorithm_parameters"]
if "algorithm_parameters" in self._train_conf:
for key, value in algo_param.items():
if key not in self._train_conf["algorithm_parameters"]:
self._train_conf["algorithm_parameters"][key] = value
else:
self._train_conf["algorithm_parameters"] = algo_param
if "role_parameters" in predict_pipeline_conf:
role_param = predict_pipeline_conf["role_parameters"]
for cpn in self._train_dsl["components"]:
for role, param in role_param.items():
for idx in param:
if param[idx].get(cpn) is not None:
del predict_pipeline_conf["role_parameters"][role][idx][cpn]
if "role_parameters" not in self._train_conf:
self._train_conf["role_parameters"] = {}
self._train_conf["role_parameters"] = tools.merge_dict(self._train_conf["role_parameters"],
predict_pipeline_conf["role_parameters"])
self._predict_dsl = tools.merge_dict(predict_pipeline_dsl, self._train_dsl)
for data_field, val in data_info.items():
cpn = data_field.split(".", -1)[0]
dataset = data_field.split(".", -1)[1]
if not isinstance(val, list):
val = [val]
self._predict_dsl["components"][cpn]["input"]["data"][dataset] = val
def _construct_train_conf(self):
self._train_conf["dsl_version"] = VERSION
self._train_conf["initiator"] = self._get_initiator_conf()
self._train_conf["role"] = self._roles
self._train_conf["job_parameters"] = {"common": {"job_type": "train"}}
for name, component in self._components.items():
param_conf = component.get_config(version=VERSION,
roles=self._roles)
if "common" in param_conf:
common_param_conf = param_conf["common"]
if "component_parameters" not in self._train_conf:
self._train_conf["component_parameters"] = {}
if "common" not in self._train_conf["component_parameters"]:
self._train_conf["component_parameters"]["common"] = {}
self._train_conf["component_parameters"]["common"].update(
common_param_conf)
if "role" in param_conf:
role_param_conf = param_conf["role"]
if "component_parameters" not in self._train_conf:
self._train_conf["component_parameters"] = {}
if "role" not in self._train_conf["component_parameters"]:
self._train_conf["component_parameters"]["role"] = {}
self._train_conf["component_parameters"][
"role"] = tools.merge_dict(
role_param_conf,
self._train_conf["component_parameters"]["role"])
LOGGER.debug(
f"self._train_conf: \n {json.dumps(self._train_conf, indent=4, ensure_ascii=False)}"
)
return self._train_conf
def _construct_train_conf(self):
self._train_conf["initiator"] = self._get_initiator_conf()
self._train_conf["role"] = self._roles
self._train_conf["job_parameters"] = self._get_job_parameters(job_type="train", version=2)
self._train_conf["role_parameters"] = {}
for name, component in self._components.items():
param_conf = component.get_config(version=VERSION, roles=self._roles)
if "algorithm_parameters" in param_conf:
algorithm_param_conf = param_conf["algorithm_parameters"]
if "algorithm_parameters" not in self._train_conf:
self._train_conf["algorithm_parameters"] = {}
self._train_conf["algorithm_parameters"].update(algorithm_param_conf)
if "role_parameters" in param_conf:
role_param_conf = param_conf["role_parameters"]
self._train_conf["role_parameters"] = tools.merge_dict(role_param_conf,
self._train_conf["role_parameters"])
# pprint.pprint(self._train_conf)
LOGGER.debug(f"self._train_conf: \n {json.dumps(self._train_conf, indent=4, ensure_ascii=False)}")
return self._train_conf