def submit_job(self, dsl=None, submit_conf=None): dsl_path = None with tempfile.TemporaryDirectory() as job_dir: if dsl: dsl_path = os.path.join(job_dir, "job_dsl.json") # pprint.pprint(dsl) LOGGER.debug( f"submit dsl is: \n {json.dumps(dsl, indent=4, ensure_ascii=False)}" ) with open(dsl_path, "w") as fout: fout.write(json.dumps(dsl)) LOGGER.debug( f"submit conf is: \n {json.dumps(submit_conf, indent=4, ensure_ascii=False)}" ) submit_path = os.path.join(job_dir, "job_runtime_conf.json") with open(submit_path, "w") as fout: fout.write(json.dumps(submit_conf)) result = self.client.job.submit(conf_path=submit_path, dsl_path=dsl_path) try: if 'retcode' not in result or result["retcode"] != 0: raise ValueError(f"retcode err") if "jobId" not in result: raise ValueError(f"jobID not in result: {result}") job_id = result["jobId"] data = result["data"] except ValueError: raise ValueError( "job submit failed, err msg: {}".format(result)) return job_id, data
def get_party_instance(self, role="guest", party_id=None) -> 'Component': if role not in ["guest", "host", "arbiter"]: raise ValueError("Role should be one of guest/host/arbiter") if party_id is not None: if isinstance(party_id, list): for _id in party_id: if not isinstance(_id, int) or _id <= 0: raise ValueError("party id should be positive integer") elif not isinstance(party_id, int) or party_id <= 0: raise ValueError("party id should be positive integer") if role not in self.__party_instance: self.__party_instance[role] = {} self.__party_instance[role]["party"] = {} party_key = party_id if isinstance(party_id, list): party_key = "|".join(map(str, party_id)) if party_key not in self.__party_instance[role]["party"]: self.__party_instance[role]["party"][party_key] = None if not self.__party_instance[role]["party"][party_key]: party_instance = copy.deepcopy(self) self._decrease_instance_count() self.__party_instance[role]["party"][party_key] = party_instance # print ("enter init") LOGGER.debug(f"enter init") return self.__party_instance[role]["party"][party_key]
def _feed_job_parameters(conf, backend=Backend.EGGROLL, work_mode=WorkMode.STANDALONE, job_type=None, model_info=None, **kwargs): submit_conf = copy.deepcopy(conf) # print("submit conf' type {}".format(type(submit_conf))) LOGGER.debug(f"submit conf type is {type(submit_conf)}") #if not isinstance(work_mode, int): # work_mode = work_mode.value #if not isinstance(backend, int): # backend = backend.value submit_conf["job_parameters"] = { "work_mode": work_mode, "backend": backend, "dsl_version": VERSION } if job_type is not None: submit_conf["job_parameters"]["job_type"] = job_type if model_info is not None: submit_conf["job_parameters"]["model_id"] = model_info.model_id submit_conf["job_parameters"]["model_version"] = model_info.model_version if kwargs: submit_conf["job_parameters"].update(kwargs) return submit_conf
def fit(self, job_parameters=None): if self._stage == "predict": raise ValueError( "This pipeline is constructed for predicting, cannot use fit interface" ) if job_parameters and not isinstance(job_parameters, JobParameters): raise ValueError( "input parameter of fit function should be JobParameters object" ) LOGGER.debug( f"in fit, _train_conf is: \n {json.dumps(self._train_conf)}") self._set_state("fit") training_conf = self._feed_job_parameters( self._train_conf, job_type="train", job_parameters=job_parameters) self._train_conf = training_conf LOGGER.debug( f"train_conf is: \n {json.dumps(training_conf, indent=4, ensure_ascii=False)}" ) self._train_job_id, detail_info = self._job_invoker.submit_job( self._train_dsl, training_conf) self._train_board_url = detail_info["board_url"] self._model_info = SimpleNamespace( model_id=detail_info["model_info"]["model_id"], model_version=detail_info["model_info"]["model_version"]) self._fit_status = self._job_invoker.monitor_job_status( self._train_job_id, self._initiator.role, self._initiator.party_id)
def fit(self, backend=Backend.EGGROLL, work_mode=WorkMode.STANDALONE, computing_engine=None, federation_engine=None, storage_engine=None, engines_address=None, federated_mode=None, federation_info=None, task_parallelism=None, federated_status_collect_type=None, federated_data_exchange_type=None, timeout=None, eggroll_run=None, spark_run=None, adaptation_parameters=None, **kwargs): if self._stage == "predict": raise ValueError("This pipeline is constructed for predicting, cannot use fit interface") job_parameters = self._filter_job_parameters(locals().copy(), kwargs) # print("_train_conf {}".format(self._train_conf)) LOGGER.debug(f"in fit, _train_conf is: \n {json.dumps(self._train_conf)}") self._set_state("fit") training_conf = self._feed_job_parameters(self._train_conf, **job_parameters) self._train_conf = training_conf LOGGER.debug(f"train_conf is: \n {json.dumps(training_conf, indent=4, ensure_ascii=False)}") self._train_job_id, detail_info = self._job_invoker.submit_job(self._train_dsl, training_conf) self._train_board_url = detail_info["board_url"] self._model_info = SimpleNamespace(model_id=detail_info["model_info"]["model_id"], model_version=detail_info["model_info"]["model_version"]) self._fit_status = self._job_invoker.monitor_job_status(self._train_job_id, self._initiator.role, self._initiator.party_id)
def _construct_train_conf(self): self._train_conf["dsl_version"] = VERSION self._train_conf["initiator"] = self._get_initiator_conf() self._train_conf["role"] = self._roles self._train_conf["job_parameters"] = {"common": {"job_type": "train"}} for name, component in self._components.items(): param_conf = component.get_config(version=VERSION, roles=self._roles) if "common" in param_conf: common_param_conf = param_conf["common"] if "component_parameters" not in self._train_conf: self._train_conf["component_parameters"] = {} if "common" not in self._train_conf["component_parameters"]: self._train_conf["component_parameters"]["common"] = {} self._train_conf["component_parameters"]["common"].update( common_param_conf) if "role" in param_conf: role_param_conf = param_conf["role"] if "component_parameters" not in self._train_conf: self._train_conf["component_parameters"] = {} if "role" not in self._train_conf["component_parameters"]: self._train_conf["component_parameters"]["role"] = {} self._train_conf["component_parameters"][ "role"] = tools.merge_dict( role_param_conf, self._train_conf["component_parameters"]["role"]) LOGGER.debug( f"self._train_conf: \n {json.dumps(self._train_conf, indent=4, ensure_ascii=False)}" ) return self._train_conf
def __init__(self, *args, **kwargs): LOGGER.debug(f"kwargs: {kwargs}") if "name" in kwargs: self._component_name = kwargs["name"] self.__party_instance = {} self._component_parameter_keywords = set(kwargs.keys()) self._role_parameter_keywords = set() self._module_name = None self._component_param = {}
def upload(self, drop=0): for data_conf in self._upload_conf: upload_conf = self._construct_upload_conf(data_conf) LOGGER.debug(f"upload_conf is {json.dumps(upload_conf)}") self._train_job_id, detail_info = self._job_invoker.upload_data( upload_conf, int(drop)) self._train_board_url = detail_info["board_url"] self._job_invoker.monitor_job_status(self._train_job_id, "local", 0)
def upload(self, backend=Backend.EGGROLL, work_mode=WorkMode.STANDALONE, drop=0): for data_conf in self._upload_conf: upload_conf = self._construct_upload_conf(data_conf, backend, work_mode) LOGGER.debug(f"upload_conf is {json.dumps(upload_conf)}") self._train_job_id, detail_info = self._job_invoker.upload_data(upload_conf, int(drop)) self._train_board_url = detail_info["board_url"] self._job_invoker.monitor_job_status(self._train_job_id, "local", 0)
def __init__(self, **kwargs): FateComponent.__init__(self, **kwargs) # print(self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) SBTTransformerParam.__init__(self, **new_kwargs) self.input = Input(self.name, data_type="multi") self.output = Output(self.name) self._module_name = "SBTFeatureTransformer"
def __init__(self, **kwargs): FateComponent.__init__(self, **kwargs) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) ScorecardParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name, data_type='single', has_model=False) self._module_name = "Scorecard"
def __init__(self, **kwargs): FateComponent.__init__(self, **kwargs) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) FeldmanVerifiableSumParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name, has_model=False) self._module_name = "FeldmanVerifiableSum"
def __init__(self, **kwargs): FateComponent.__init__(self, **kwargs) # print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) SampleParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name, has_model=False) self._module_name = "FederatedSample"
def __init__(self, **kwargs): Component.__init__(self, **kwargs) # print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) DataSplitParam.__init__(self, **new_kwargs) self.input = Input(self.name, ) self.output = Output(self.name, has_model=False, data_type="multi") self._module_name = "HeteroDataSplit"
def __init__(self, **kwargs): Component.__init__(self, **kwargs) # print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) HomoOneHotParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name) self._module_name = "HomoOneHotEncoder"
def __init__(self, **kwargs): FateComponent.__init__(self, **kwargs) # print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) HeteroLogisticParam.__init__(self, **new_kwargs) self.input = Input(self.name, data_type="multi") self.output = Output(self.name) self._module_name = "HeteroLR"
def __init__(self, **kwargs): Component.__init__(self, **kwargs) # print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) LocalBaselineParam.__init__(self, **new_kwargs) self.input = Input(self.name, data_type="multi") self.output = Output(self.name) self._module_name = "LocalBaseline"
def __init__(self, **kwargs): FateComponent.__init__(self, **kwargs) #print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) FeatureImputationParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name, has_model=True) self._module_name = "FeatureImputation"
def __init__(self, **kwargs): Component.__init__(self, **kwargs) # print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) IntersectParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name, has_model=False) self._module_name = "Intersection"
def __init__(self, **kwargs): FateComponent.__init__(self, **kwargs) # print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) KmeansParam.__init__(self, **new_kwargs) self.input = Input(self.name, data_type="multi") self.output = Output(self.name, data_type="no_limit", output_unit=2) self._module_name = "HeteroKmeans"
def __init__(self, **kwargs): FateComponent.__init__(self, **kwargs) #print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) StatisticsParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name, has_model=True, has_data=False) self._module_name = "DataStatistics"
def __init__(self, **kwargs): Component.__init__(self, **kwargs) # print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) FeatureBinningParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name) self._module_name = "HeteroFeatureBinning"
def __init__(self, **kwargs): Component.__init__(self, **kwargs) #print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) DataTransformParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name, data_type='single') self._module_name = "DataTransform"
def __init__(self, **kwargs): FateComponent.__init__(self, **kwargs) # print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) SecureInformationRetrievalParam.__init__(self, **new_kwargs) self.input = Input(self.name, data_type="single") self.output = Output(self.name) self._module_name = "SecureInformationRetrieval"
def __init__(self, *args, **kwargs): LOGGER.debug(f"kwargs: {kwargs}") if "name" in kwargs: self._component_name = kwargs["name"] self.__party_instance = {} self._component_parameter_keywords = set(kwargs.keys()) self._role_parameter_keywords = set() self._module_name = None self._component_param = {} self._provider = None # deprecated, to compatible with fate-1.7.0 self._source_provider = None self._provider_version = None
def __init__(self, **kwargs): Component.__init__(self, **kwargs) #print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) ColumnExpandParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name, data_type='single', has_model=True) self._module_name = "ColumnExpand"
def __init__(self, **kwargs): FateComponent.__init__(self, **kwargs) # print(self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) HeteroSecureBoostParam.__init__(self, **new_kwargs) self.input = Input(self.name, data_type="multi") self.output = Output(self.name) self._module_name = "HeteroFastSecureBoost" self._source_provider = ProviderType.FATE
def _run_cmd(cls, cmd, output_while_running=False): subp = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if not output_while_running: stdout, stderr = subp.communicate() return stdout.decode("utf-8") else: for line in subp.stdout: if line == "": continue else: # print(line.strip()) LOGGER.debug(f"{line.strip()}")
def __init__(self, **kwargs): FateFlowComponent.__init__(self, **kwargs) # print (self.name) LOGGER.debug(f"{self.name} component created") new_kwargs = self.erase_component_base_param(**kwargs) CacheLoaderParam.__init__(self, **new_kwargs) self.input = Input(self.name) self.output = Output(self.name, has_model=False, has_cache=True, has_data=False) self._module_name = "CacheLoader"
def _feed_job_parameters(self, conf, job_type=None, model_info=None, job_parameters=None): submit_conf = copy.deepcopy(conf) LOGGER.debug(f"submit conf type is {type(submit_conf)}") if job_parameters: submit_conf["job_parameters"] = job_parameters.get_config(roles=self._roles) if "common" not in submit_conf["job_parameters"]: submit_conf["job_parameters"]["common"] = {} submit_conf["job_parameters"]["common"]["job_type"] = job_type if model_info is not None: submit_conf["job_parameters"]["common"]["model_id"] = model_info.model_id submit_conf["job_parameters"]["common"]["model_version"] = model_info.model_version return submit_conf