def submit_job(self, dsl=None, submit_conf=None):
dsl_path = None
with tempfile.TemporaryDirectory() as job_dir:
if dsl:
dsl_path = os.path.join(job_dir, "job_dsl.json")
# pprint.pprint(dsl)
LOGGER.debug(
f"submit dsl is: \n {json.dumps(dsl, indent=4, ensure_ascii=False)}"
)
with open(dsl_path, "w") as fout:
fout.write(json.dumps(dsl))
LOGGER.debug(
f"submit conf is: \n {json.dumps(submit_conf, indent=4, ensure_ascii=False)}"
)
submit_path = os.path.join(job_dir, "job_runtime_conf.json")
with open(submit_path, "w") as fout:
fout.write(json.dumps(submit_conf))
result = self.client.job.submit(conf_path=submit_path,
dsl_path=dsl_path)
try:
if 'retcode' not in result or result["retcode"] != 0:
raise ValueError(f"retcode err")
if "jobId" not in result:
raise ValueError(f"jobID not in result: {result}")
job_id = result["jobId"]
data = result["data"]
except ValueError:
raise ValueError(
"job submit failed, err msg: {}".format(result))
return job_id, data
def get_party_instance(self, role="guest", party_id=None) -> 'Component':
if role not in ["guest", "host", "arbiter"]:
raise ValueError("Role should be one of guest/host/arbiter")
if party_id is not None:
if isinstance(party_id, list):
for _id in party_id:
if not isinstance(_id, int) or _id <= 0:
raise ValueError("party id should be positive integer")
elif not isinstance(party_id, int) or party_id <= 0:
raise ValueError("party id should be positive integer")
if role not in self.__party_instance:
self.__party_instance[role] = {}
self.__party_instance[role]["party"] = {}
party_key = party_id
if isinstance(party_id, list):
party_key = "|".join(map(str, party_id))
if party_key not in self.__party_instance[role]["party"]:
self.__party_instance[role]["party"][party_key] = None
if not self.__party_instance[role]["party"][party_key]:
party_instance = copy.deepcopy(self)
self._decrease_instance_count()
self.__party_instance[role]["party"][party_key] = party_instance
# print ("enter init")
LOGGER.debug(f"enter init")
return self.__party_instance[role]["party"][party_key]
def _feed_job_parameters(conf, backend=Backend.EGGROLL, work_mode=WorkMode.STANDALONE, job_type=None, model_info=None, **kwargs):
submit_conf = copy.deepcopy(conf)
# print("submit conf' type {}".format(type(submit_conf)))
LOGGER.debug(f"submit conf type is {type(submit_conf)}")
#if not isinstance(work_mode, int):
# work_mode = work_mode.value
#if not isinstance(backend, int):
# backend = backend.value
submit_conf["job_parameters"] = {
"work_mode": work_mode,
"backend": backend,
"dsl_version": VERSION
}
if job_type is not None:
submit_conf["job_parameters"]["job_type"] = job_type
if model_info is not None:
submit_conf["job_parameters"]["model_id"] = model_info.model_id
submit_conf["job_parameters"]["model_version"] = model_info.model_version
if kwargs:
submit_conf["job_parameters"].update(kwargs)
return submit_conf
def fit(self, job_parameters=None):
if self._stage == "predict":
raise ValueError(
"This pipeline is constructed for predicting, cannot use fit interface"
)
if job_parameters and not isinstance(job_parameters, JobParameters):
raise ValueError(
"input parameter of fit function should be JobParameters object"
)
LOGGER.debug(
f"in fit, _train_conf is: \n {json.dumps(self._train_conf)}")
self._set_state("fit")
training_conf = self._feed_job_parameters(
self._train_conf, job_type="train", job_parameters=job_parameters)
self._train_conf = training_conf
LOGGER.debug(
f"train_conf is: \n {json.dumps(training_conf, indent=4, ensure_ascii=False)}"
)
self._train_job_id, detail_info = self._job_invoker.submit_job(
self._train_dsl, training_conf)
self._train_board_url = detail_info["board_url"]
self._model_info = SimpleNamespace(
model_id=detail_info["model_info"]["model_id"],
model_version=detail_info["model_info"]["model_version"])
self._fit_status = self._job_invoker.monitor_job_status(
self._train_job_id, self._initiator.role, self._initiator.party_id)
def fit(self, backend=Backend.EGGROLL, work_mode=WorkMode.STANDALONE, computing_engine=None,
federation_engine=None, storage_engine=None, engines_address=None, federated_mode=None,
federation_info=None, task_parallelism=None, federated_status_collect_type=None,
federated_data_exchange_type=None, timeout=None, eggroll_run=None, spark_run=None,
adaptation_parameters=None, **kwargs):
if self._stage == "predict":
raise ValueError("This pipeline is constructed for predicting, cannot use fit interface")
job_parameters = self._filter_job_parameters(locals().copy(), kwargs)
# print("_train_conf {}".format(self._train_conf))
LOGGER.debug(f"in fit, _train_conf is: \n {json.dumps(self._train_conf)}")
self._set_state("fit")
training_conf = self._feed_job_parameters(self._train_conf, **job_parameters)
self._train_conf = training_conf
LOGGER.debug(f"train_conf is: \n {json.dumps(training_conf, indent=4, ensure_ascii=False)}")
self._train_job_id, detail_info = self._job_invoker.submit_job(self._train_dsl, training_conf)
self._train_board_url = detail_info["board_url"]
self._model_info = SimpleNamespace(model_id=detail_info["model_info"]["model_id"],
model_version=detail_info["model_info"]["model_version"])
self._fit_status = self._job_invoker.monitor_job_status(self._train_job_id,
self._initiator.role,
self._initiator.party_id)
def _construct_train_conf(self):
self._train_conf["dsl_version"] = VERSION
self._train_conf["initiator"] = self._get_initiator_conf()
self._train_conf["role"] = self._roles
self._train_conf["job_parameters"] = {"common": {"job_type": "train"}}
for name, component in self._components.items():
param_conf = component.get_config(version=VERSION,
roles=self._roles)
if "common" in param_conf:
common_param_conf = param_conf["common"]
if "component_parameters" not in self._train_conf:
self._train_conf["component_parameters"] = {}
if "common" not in self._train_conf["component_parameters"]:
self._train_conf["component_parameters"]["common"] = {}
self._train_conf["component_parameters"]["common"].update(
common_param_conf)
if "role" in param_conf:
role_param_conf = param_conf["role"]
if "component_parameters" not in self._train_conf:
self._train_conf["component_parameters"] = {}
if "role" not in self._train_conf["component_parameters"]:
self._train_conf["component_parameters"]["role"] = {}
self._train_conf["component_parameters"][
"role"] = tools.merge_dict(
role_param_conf,
self._train_conf["component_parameters"]["role"])
LOGGER.debug(
f"self._train_conf: \n {json.dumps(self._train_conf, indent=4, ensure_ascii=False)}"
)
return self._train_conf
def __init__(self, *args, **kwargs):
LOGGER.debug(f"kwargs: {kwargs}")
if "name" in kwargs:
self._component_name = kwargs["name"]
self.__party_instance = {}
self._component_parameter_keywords = set(kwargs.keys())
self._role_parameter_keywords = set()
self._module_name = None
self._component_param = {}
def upload(self, drop=0):
for data_conf in self._upload_conf:
upload_conf = self._construct_upload_conf(data_conf)
LOGGER.debug(f"upload_conf is {json.dumps(upload_conf)}")
self._train_job_id, detail_info = self._job_invoker.upload_data(
upload_conf, int(drop))
self._train_board_url = detail_info["board_url"]
self._job_invoker.monitor_job_status(self._train_job_id, "local",
0)
def upload(self, backend=Backend.EGGROLL, work_mode=WorkMode.STANDALONE, drop=0):
for data_conf in self._upload_conf:
upload_conf = self._construct_upload_conf(data_conf, backend, work_mode)
LOGGER.debug(f"upload_conf is {json.dumps(upload_conf)}")
self._train_job_id, detail_info = self._job_invoker.upload_data(upload_conf, int(drop))
self._train_board_url = detail_info["board_url"]
self._job_invoker.monitor_job_status(self._train_job_id,
"local",
0)
def __init__(self, **kwargs):
FateComponent.__init__(self, **kwargs)
# print(self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
SBTTransformerParam.__init__(self, **new_kwargs)
self.input = Input(self.name, data_type="multi")
self.output = Output(self.name)
self._module_name = "SBTFeatureTransformer"
def __init__(self, **kwargs):
FateComponent.__init__(self, **kwargs)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
ScorecardParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name, data_type='single', has_model=False)
self._module_name = "Scorecard"
def __init__(self, **kwargs):
FateComponent.__init__(self, **kwargs)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
FeldmanVerifiableSumParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name, has_model=False)
self._module_name = "FeldmanVerifiableSum"
def __init__(self, **kwargs):
FateComponent.__init__(self, **kwargs)
# print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
SampleParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name, has_model=False)
self._module_name = "FederatedSample"
def __init__(self, **kwargs):
Component.__init__(self, **kwargs)
# print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
DataSplitParam.__init__(self, **new_kwargs)
self.input = Input(self.name, )
self.output = Output(self.name, has_model=False, data_type="multi")
self._module_name = "HeteroDataSplit"
def __init__(self, **kwargs):
Component.__init__(self, **kwargs)
# print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
HomoOneHotParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name)
self._module_name = "HomoOneHotEncoder"
def __init__(self, **kwargs):
FateComponent.__init__(self, **kwargs)
# print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
HeteroLogisticParam.__init__(self, **new_kwargs)
self.input = Input(self.name, data_type="multi")
self.output = Output(self.name)
self._module_name = "HeteroLR"
def __init__(self, **kwargs):
Component.__init__(self, **kwargs)
# print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
LocalBaselineParam.__init__(self, **new_kwargs)
self.input = Input(self.name, data_type="multi")
self.output = Output(self.name)
self._module_name = "LocalBaseline"
def __init__(self, **kwargs):
FateComponent.__init__(self, **kwargs)
#print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
FeatureImputationParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name, has_model=True)
self._module_name = "FeatureImputation"
def __init__(self, **kwargs):
Component.__init__(self, **kwargs)
# print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
IntersectParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name, has_model=False)
self._module_name = "Intersection"
def __init__(self, **kwargs):
FateComponent.__init__(self, **kwargs)
# print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
KmeansParam.__init__(self, **new_kwargs)
self.input = Input(self.name, data_type="multi")
self.output = Output(self.name, data_type="no_limit", output_unit=2)
self._module_name = "HeteroKmeans"
def __init__(self, **kwargs):
FateComponent.__init__(self, **kwargs)
#print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
StatisticsParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name, has_model=True, has_data=False)
self._module_name = "DataStatistics"
def __init__(self, **kwargs):
Component.__init__(self, **kwargs)
# print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
FeatureBinningParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name)
self._module_name = "HeteroFeatureBinning"
def __init__(self, **kwargs):
Component.__init__(self, **kwargs)
#print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
DataTransformParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name, data_type='single')
self._module_name = "DataTransform"
def __init__(self, **kwargs):
FateComponent.__init__(self, **kwargs)
# print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
SecureInformationRetrievalParam.__init__(self, **new_kwargs)
self.input = Input(self.name, data_type="single")
self.output = Output(self.name)
self._module_name = "SecureInformationRetrieval"
def __init__(self, *args, **kwargs):
LOGGER.debug(f"kwargs: {kwargs}")
if "name" in kwargs:
self._component_name = kwargs["name"]
self.__party_instance = {}
self._component_parameter_keywords = set(kwargs.keys())
self._role_parameter_keywords = set()
self._module_name = None
self._component_param = {}
self._provider = None # deprecated, to compatible with fate-1.7.0
self._source_provider = None
self._provider_version = None
def __init__(self, **kwargs):
Component.__init__(self, **kwargs)
#print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
ColumnExpandParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name, data_type='single', has_model=True)
self._module_name = "ColumnExpand"
def __init__(self, **kwargs):
FateComponent.__init__(self, **kwargs)
# print(self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
HeteroSecureBoostParam.__init__(self, **new_kwargs)
self.input = Input(self.name, data_type="multi")
self.output = Output(self.name)
self._module_name = "HeteroFastSecureBoost"
self._source_provider = ProviderType.FATE
def _run_cmd(cls, cmd, output_while_running=False):
subp = subprocess.Popen(cmd,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if not output_while_running:
stdout, stderr = subp.communicate()
return stdout.decode("utf-8")
else:
for line in subp.stdout:
if line == "":
continue
else:
# print(line.strip())
LOGGER.debug(f"{line.strip()}")
def __init__(self, **kwargs):
FateFlowComponent.__init__(self, **kwargs)
# print (self.name)
LOGGER.debug(f"{self.name} component created")
new_kwargs = self.erase_component_base_param(**kwargs)
CacheLoaderParam.__init__(self, **new_kwargs)
self.input = Input(self.name)
self.output = Output(self.name,
has_model=False,
has_cache=True,
has_data=False)
self._module_name = "CacheLoader"
def _feed_job_parameters(self, conf, job_type=None,
model_info=None, job_parameters=None):
submit_conf = copy.deepcopy(conf)
LOGGER.debug(f"submit conf type is {type(submit_conf)}")
if job_parameters:
submit_conf["job_parameters"] = job_parameters.get_config(roles=self._roles)
if "common" not in submit_conf["job_parameters"]:
submit_conf["job_parameters"]["common"] = {}
submit_conf["job_parameters"]["common"]["job_type"] = job_type
if model_info is not None:
submit_conf["job_parameters"]["common"]["model_id"] = model_info.model_id
submit_conf["job_parameters"]["common"]["model_version"] = model_info.model_version
return submit_conf
