def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] guest_train_data = [{"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"}, {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"}] pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data[0]) reader_1 = Reader(name="reader_1") reader_1.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data[1]) union_0 = Union(name="union_0", allow_missing=False, keep_duplicate=True) data_transform_0 = DataTransform(name="data_transform_0", with_label=True, output_format="dense", label_name="y", missing_fill=False, outlier_replace=False) pipeline.add_component(reader_0) pipeline.add_component(reader_1) pipeline.add_component(union_0, data=Data(data=[reader_0.output.data, reader_1.output.data])) pipeline.add_component(data_transform_0, data=Data(data=union_0.output.data)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] hosts = parties.host arbiter = parties.arbiter[0] guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = [{"name": "motor_hetero_host", "namespace": f"experiment{namespace}"}, {"name": "motor_hetero_host", "namespace": f"experiment{namespace}"}] pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=hosts, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=hosts[0]).component_param(table=host_train_data[0]) reader_0.get_party_instance(role='host', party_id=hosts[1]).component_param(table=host_train_data[1]) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed", label_type="float", output_format="dense") data_transform_0.get_party_instance(role='host', party_id=hosts).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0") hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, encrypted_mode_calculator_param={"mode": "fast"}) evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) # evaluation_0.get_party_instance(role='host', party_id=hosts[0]).component_param(need_run=False) # evaluation_0.get_party_instance(role='host', party_id=hosts[1]).component_param(need_run=False) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data)) pipeline.add_component(evaluation_0, data=Data(data=hetero_linr_0.output.data)) pipeline.compile() pipeline.fit() # predict # deploy required components pipeline.deploy_component([data_transform_0, intersection_0, hetero_linr_0]) predict_pipeline = PipeLine() # add data reader onto predict pipeline predict_pipeline.add_component(reader_0) # add selected components from train pipeline onto predict pipeline # specify data source predict_pipeline.add_component(pipeline, data=Data(predict_input={pipeline.data_transform_0.input.data: reader_0.output.data})) # run predict model predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = {"name": "tag_value_1000_140", "namespace": f"experiment{namespace}"} host_train_data = {"name": "tag_value_1000_140", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) reader_1 = Reader(name="reader_1") reader_1.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_1.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0", input_format="tag", tag_with_value=True, with_match_id=True) data_transform_1 = DataTransform(name="data_transform_1") pipeline.add_component(reader_0) pipeline.add_component(reader_1) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data), model=Model(model=data_transform_0.output.model)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment_sid{namespace}"} host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment_sid{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0", with_label=False, with_match_id=True) intersection_0 = Intersection(name="intersection_0") feature_imputation_0 = FeatureImputation(name="feature_imputation_0", missing_fill_method="designated", default_value=42, missing_impute=[0]) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(feature_imputation_0, data=Data(data=intersection_0.output.data)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] guest_train_data = { "name": "breast_homo_guest", "namespace": f"experiment{namespace}" } host_train_data = { "name": "breast_homo_host", "namespace": f"experiment{namespace}" } # initialize pipeline pipeline = PipeLine() # set job initiator pipeline.set_initiator(role='guest', party_id=guest) # set participants information pipeline.set_roles(guest=guest, host=host, arbiter=arbiter) # define Reader components to read in data reader_0 = Reader(name="reader_0") # configure Reader for guest reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) # configure Reader for host reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) # define DataTransform components data_transform_0 = DataTransform( name="data_transform_0", with_label=True, output_format="dense") # start component numbering at 0 homo_binning_0 = HomoFeatureBinning(name='homo_binning_0', sample_bins=1000) homo_binning_1 = HomoFeatureBinning(name='homo_binning_1', sample_bins=1000) # add components to pipeline, in order of task execution pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) # set data input sources of intersection components pipeline.add_component(homo_binning_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(homo_binning_1, data=Data(data=data_transform_0.output.data), model=Model(model=homo_binning_0.output.model)) # compile pipeline once finished adding modules, this step will form conf and dsl files for running job pipeline.compile() # fit model pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] backend = config.backend work_mode = config.work_mode guest_train_data = {"name": "dvisits_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "dvisits_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).algorithm_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).algorithm_param(table=host_train_data) dataio_0 = DataIO(name="dataio_0") dataio_0.get_party_instance(role='guest', party_id=guest).algorithm_param(with_label=True, label_name="doctorco", label_type="float", output_format="dense") dataio_0.get_party_instance(role='host', party_id=host).algorithm_param(with_label=False) intersection_0 = Intersection(name="intersection_0") hetero_poisson_0 = HeteroPoisson(name="hetero_poisson_0", early_stop="weight_diff", max_iter=10, alpha=100.0, batch_size=-1, learning_rate=0.01, optimizer="rmsprop", exposure_colname="exposure", decay_sqrt=False, tol=0.001, init_param={"init_method": "zeros"}, penalty="L2", encrypted_mode_calculator_param={"mode": "fast"}) evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) evaluation_0.get_party_instance(role='host', party_id=host).algorithm_param(need_run=False) pipeline.add_component(reader_0) pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data)) pipeline.add_component(hetero_poisson_0, data=Data(train_data=intersection_0.output.data)) pipeline.add_component(evaluation_0, data=Data(data=hetero_poisson_0.output.data)) pipeline.compile() pipeline.fit(backend=backend, work_mode=work_mode) # predict # deploy required components pipeline.deploy_component([dataio_0, intersection_0, hetero_poisson_0]) predict_pipeline = PipeLine() # add data reader onto predict pipeline predict_pipeline.add_component(reader_0) # add selected components from train pipeline onto predict pipeline # specify data source predict_pipeline.add_component(pipeline, data=Data(predict_input={pipeline.dataio_0.input.data: reader_0.output.data})) # run predict model predict_pipeline.predict(backend=backend, work_mode=work_mode)
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = { "name": "breast_hetero_guest", "namespace": f"experiment{namespace}" } host_train_data = { "name": "breast_hetero_host", "namespace": f"experiment{namespace}" } pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance( role='guest', party_id=guest).component_param(with_label=False, output_format="dense") data_transform_0.get_party_instance( role='host', party_id=host).component_param(with_label=False, output_format="dense") param = { "intersect_method": "raw", "sync_intersect_ids": True, "only_output_key": True, "raw_params": { "use_hash": True, "hash_method": "sm3", "salt": "12345", "base64": False, "join_role": "host" } } intersect_0 = Intersection(name="intersect_0", **param) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True) data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0") hetero_nn_0 = HeteroNN(name="hetero_nn_0", epochs=100, interactive_layer_lr=0.15, batch_size=-1, early_stop="diff", selector_param={"method": "relative"}) guest_nn_0 = hetero_nn_0.get_party_instance(role='guest', party_id=guest) guest_nn_0.add_bottom_model(Dense(units=3, input_shape=(10,), activation="relu", kernel_initializer=initializers.Constant(value=1))) guest_nn_0.set_interactve_layer(Dense(units=2, input_shape=(2,), kernel_initializer=initializers.Constant(value=1))) guest_nn_0.add_top_model(Dense(units=1, input_shape=(2,), activation="sigmoid", kernel_initializer=initializers.Constant(value=1))) host_nn_0 = hetero_nn_0.get_party_instance(role='host', party_id=host) host_nn_0.add_bottom_model(Dense(units=3, input_shape=(20,), activation="relu", kernel_initializer=initializers.Constant(value=1))) host_nn_0.set_interactve_layer(Dense(units=2, input_shape=(2,), kernel_initializer=initializers.Constant(value=1))) hetero_nn_0.compile(optimizer=optimizers.SGD(lr=0.15), loss="binary_crossentropy") hetero_nn_1 = HeteroNN(name="hetero_nn_1") evaluation_0 = Evaluation(name="evaluation_0") pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(hetero_nn_0, data=Data(train_data=intersection_0.output.data)) pipeline.add_component(hetero_nn_1, data=Data(test_data=intersection_0.output.data), model=Model(model=hetero_nn_0.output.model)) pipeline.add_component(evaluation_0, data=Data(data=hetero_nn_0.output.data)) pipeline.compile() pipeline.fit() print(hetero_nn_0.get_config(roles={"guest": [guest], "host": [host]})) print(pipeline.get_component("hetero_nn_0").get_summary())
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance( role='guest', party_id=guest).component_param( with_label=False, output_format="dense") data_transform_0.get_party_instance( role='host', party_id=host).component_param( with_label=False, output_format="dense") param_0 = { "intersect_method": "rsa", "rsa_params": { "hash_method": "sha256", "final_hash_method": "sha256", "key_length": 2048 }, "run_cache": True } param_1 = { "intersect_method": "rsa", "sync_intersect_ids": False, "only_output_key": True, "rsa_params": { "hash_method": "sha256", "final_hash_method": "sha256", "key_length": 2048 } } intersect_0 = Intersection(name="intersect_0", **param_0) intersect_1 = Intersection(name="intersect_1", **param_1) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component( intersect_1, data=Data( data=data_transform_0.output.data), cache=Cache( intersect_0.output.cache)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] # data sets guest_train_data = {"name": "student_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "student_hetero_host", "namespace": f"experiment{namespace}"} # init pipeline pipeline = PipeLine().set_initiator(role="guest", party_id=guest).set_roles(guest=guest, host=host,) # set data reader and data-io reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance( role="guest", party_id=guest).component_param( with_label=True, output_format="dense", label_type="float") data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False) # data intersect component intersect_0 = Intersection(name="intersection_0") # secure boost component hetero_secure_boost_0 = HeteroSecureBoost(name="hetero_secure_boost_0", num_trees=3, task_type="regression", objective_param={"objective": "lse"}, encrypt_param={"method": "Paillier"}, tree_param={"max_depth": 3}, validation_freqs=1, cv_param={ "need_cv": True, "n_splits": 5, "shuffle": False, "random_seed": 103 } ) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(hetero_secure_boost_0, data=Data(train_data=intersect_0.output.data)) pipeline.compile() pipeline.fit() print("fitting hetero secureboost done, result:") print(pipeline.get_component("hetero_secure_boost_0").get_summary())
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "motor_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance( role='guest', party_id=guest).component_param( with_label=True, label_name="motor_speed", label_type="float", output_format="dense") data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0", only_output_key=False) hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=5, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, callback_param={"callbacks": ["ModelCheckpoint"]}, init_param={"init_method": "zeros"}, floating_point_precision=23) evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) hetero_linr_1 = HeteroLinR(name="hetero_linr_1", max_iter=15, penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, floating_point_precision=23 ) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data)) pipeline.add_component(hetero_linr_1, data=Data(train_data=intersection_0.output.data), model=Model(hetero_linr_0.output.model)) pipeline.add_component(evaluation_0, data=Data(data=hetero_linr_1.output.data)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} # initialize pipeline pipeline = PipeLine() # set job initiator pipeline.set_initiator(role="guest", party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) # define Reader components to read in data reader_0 = Reader(name="reader_0") # configure Reader reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") # start component numbering at 0 data_transform_0_guest_party_instance = data_transform_0.get_party_instance(role="guest", party_id=guest) data_transform_0_guest_party_instance.component_param(with_label=True, output_format="dense") data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False, output_format="dense") intersection_0 = Intersection(name="intersection_0") label_transform_0 = LabelTransform(name="label_transform_0", label_encoder={"0": 1, "1": 0}, label_list=[0, 1]) label_transform_0.get_party_instance(role="host", party_id=host).component_param(need_run=False) hetero_lr_0 = HeteroLR(name="hetero_lr_0", penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, floating_point_precision=23) label_transform_1 = LabelTransform(name="label_transform_1") # add components to pipeline, in order of task execution pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(label_transform_0, data=Data(data=intersection_0.output.data)) pipeline.add_component(hetero_lr_0, data=Data(train_data=label_transform_0.output.data)) pipeline.add_component(label_transform_1, data=Data(data=hetero_lr_0.output.data), model=Model(label_transform_0.output.model)) # compile pipeline once finished adding modules, this step will form conf and dsl files for running job pipeline.compile() # fit model pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] backend = config.backend work_mode = config.work_mode guest_train_data = { "name": "breast_homo_guest", "namespace": f"experiment{namespace}" } host_train_data = { "name": "breast_homo_host", "namespace": f"experiment{namespace}" } pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) dataio_0 = DataIO(name="dataio_0") dataio_0.get_party_instance(role='guest', party_id=guest).component_param( with_label=True, output_format="dense", label_name="y", label_type="int") dataio_0.get_party_instance(role='host', party_id=host).component_param(with_label=True) homo_data_split_0 = HomoDataSplit(name="homo_data_split_0", stratified=True, test_size=0.3, validate_size=0.2) pipeline.add_component(reader_0) pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data)) pipeline.add_component(homo_data_split_0, data=Data(data=dataio_0.output.data)) pipeline.compile() job_parameters = JobParameters(backend=backend, work_mode=work_mode) pipeline.fit(job_parameters) print(pipeline.get_component("homo_data_split_0").get_summary())
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] backend = config.backend work_mode = config.work_mode guest_train_data = { "name": "breast_hetero_guest", "namespace": f"experiment{namespace}" } # initialize pipeline pipeline = PipeLine() # set job initiator pipeline.set_initiator(role="guest", party_id=guest).set_roles(guest=guest) # define Reader components to read in data reader_0 = Reader(name="reader_0") # configure Reader for guest reader_0.get_party_instance( role="guest", party_id=guest).component_param(table=guest_train_data) # define ColumnExpand components column_expand_0 = ColumnExpand(name="column_expand_0") column_expand_0.get_party_instance( role="guest", party_id=guest).component_param( need_run=True, method="manual", append_header=["x_0", "x_1", "x_2", "x_3"], fill_value=[0, 0.2, 0.5, 1]) # define DataIO components dataio_0 = DataIO(name="dataio_0") # start component numbering at 0 # get DataIO party instance of guest dataio_0_guest_party_instance = dataio_0.get_party_instance(role="guest", party_id=guest) # configure DataIO for guest dataio_0_guest_party_instance.component_param(with_label=True, output_format="dense") # add components to pipeline, in order of task execution pipeline.add_component(reader_0) pipeline.add_component(column_expand_0, data=Data(data=reader_0.output.data)) pipeline.add_component(dataio_0, data=Data(data=column_expand_0.output.data)) # compile pipeline once finished adding modules, this step will form conf and dsl files for running job pipeline.compile() # fit model job_parameters = JobParameters(backend=backend, work_mode=work_mode) pipeline.fit(job_parameters)
def run_homo_nn_pipeline(config, namespace, data: dict, nn_component, num_host): if isinstance(config, str): config = load_job_config(config) guest_train_data = data["guest"] host_train_data = data["host"][:num_host] for d in [guest_train_data, *host_train_data]: d["namespace"] = f"{d['namespace']}{namespace}" hosts = config.parties.host[:num_host] pipeline = (PipeLine().set_initiator( role="guest", party_id=config.parties.guest[0]).set_roles( guest=config.parties.guest[0], host=hosts, arbiter=config.parties.arbiter)) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role="guest", party_id=config.parties.guest[0]).component_param( table=guest_train_data) for i in range(num_host): reader_0.get_party_instance( role="host", party_id=hosts[i]).component_param(table=host_train_data[i]) data_transform_0 = DataTransform(name="data_transform_0", with_label=True) data_transform_0.get_party_instance( role="guest", party_id=config.parties.guest[0]).component_param( with_label=True, output_format="dense") data_transform_0.get_party_instance( role="host", party_id=hosts).component_param(with_label=True) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(nn_component, data=Data(train_data=data_transform_0.output.data)) pipeline.compile() pipeline.fit() print(pipeline.get_component("homo_nn_0").get_summary()) pipeline.deploy_component([data_transform_0, nn_component]) # predict predict_pipeline = PipeLine() predict_pipeline.add_component(reader_0) predict_pipeline.add_component( pipeline, data=Data(predict_input={ pipeline.data_transform_0.input.data: reader_0.output.data }), ) # run predict model predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] hosts = parties.host arbiter = parties.arbiter[0] guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = [{"name": "motor_hetero_host", "namespace": f"experiment{namespace}"}, {"name": "motor_hetero_host", "namespace": f"experiment{namespace}"}] pipeline = PipeLine().set_initiator( role='guest', party_id=guest).set_roles( guest=guest, host=hosts, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=hosts[0]).component_param(table=host_train_data[0]) reader_0.get_party_instance(role='host', party_id=hosts[1]).component_param(table=host_train_data[1]) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance( role='guest', party_id=guest).component_param( with_label=True, label_name="motor_speed", label_type="float", output_format="dense") data_transform_0.get_party_instance(role='host', party_id=hosts).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0") hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="None", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, cv_param={"n_splits": 5, "shuffle": False, "random_seed": 42, "need_cv": True } ) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] hosts = parties.host arbiter = parties.arbiter[0] backend = config.backend work_mode = config.work_mode guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=hosts, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=hosts).component_param(table=host_train_data) dataio_0 = DataIO(name="dataio_0") dataio_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="y", label_type="int", output_format="dense") dataio_0.get_party_instance(role='host', party_id=hosts).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0") sample_weight_0 = SampleWeight(name="sample_weight_0") sample_weight_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True, class_weight="balanced") sample_weight_0.get_party_instance(role='host', party_id=hosts).component_param(need_run=False) hetero_lr_0 = HeteroLR(name="hetero_lr_0", optimizer="nesterov_momentum_sgd", tol=0.001, alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, init_param={"init_method": "zeros"}) evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary", pos_label=1) # evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) pipeline.add_component(reader_0) pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data)) pipeline.add_component(sample_weight_0, data=Data(data=intersection_0.output.data)) pipeline.add_component(hetero_lr_0, data=Data(train_data=sample_weight_0.output.data)) pipeline.add_component(evaluation_0, data=Data(data=hetero_lr_0.output.data)) pipeline.compile() job_parameters = JobParameters(backend=backend, work_mode=work_mode) pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] backend = config.backend work_mode = config.work_mode guest_train_data = { "name": "breast_hetero_guest", "namespace": f"experiment{namespace}" } pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role='guest', party_id=guest).algorithm_param(table=guest_train_data) reader_1 = Reader(name="reader_1") reader_1.get_party_instance( role='guest', party_id=guest).algorithm_param(table=guest_train_data) dataio_0 = DataIO(name="dataio_0", with_label=True, output_format="dense", label_name="y", missing_fill=False, outlier_replace=False) dataio_1 = DataIO(name="dataio_1", with_label=True, output_format="dense", label_name="y", missing_fill=False, outlier_replace=False) union_0 = Union(name="union_0", allow_missing=False, need_run=True) pipeline.add_component(reader_0) pipeline.add_component(reader_1) pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data)) pipeline.add_component(dataio_1, data=Data(data=reader_1.output.data), model=Model(dataio_0.output.model)) pipeline.add_component( union_0, data=Data(data=[dataio_0.output.data, dataio_1.output.data])) pipeline.compile() pipeline.fit(backend=backend, work_mode=work_mode)
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] hosts = parties.host guest_train_data = {"name": "breast_homo_test", "namespace": f"experiment_sid{namespace}"} host_train_data = {"name": "breast_homo_test", "namespace": f"experiment_sid{namespace}"} # initialize pipeline pipeline = PipeLine() # set job initiator pipeline.set_initiator(role="guest", party_id=guest) # set participants information pipeline.set_roles(guest=guest, host=hosts) # define Reader components to read in data reader_0 = Reader(name="reader_0") # configure Reader for guest reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data) # configure Reader for host reader_0.get_party_instance(role="host", party_id=hosts).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0", with_match_id=True) # get and configure DataTransform party instance of guest data_transform_0.get_party_instance( role="guest", party_id=guest).component_param( with_label=False, output_format="dense") # get and configure DataTransform party instance of host data_transform_0.get_party_instance(role="host", party_id=hosts).component_param(with_label=False) # define FeldmanVerifiableSum components feldmanverifiablesum_0 = FeldmanVerifiableSum(name="feldmanverifiablesum_0") feldmanverifiablesum_0.get_party_instance(role="guest", party_id=guest).component_param(sum_cols=[1, 2, 3], q_n=6) feldmanverifiablesum_0.get_party_instance(role="host", party_id=hosts).component_param(sum_cols=[1, 2, 3], q_n=6) # add components to pipeline, in order of task execution. pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(feldmanverifiablesum_0, data=Data(data=data_transform_0.output.data)) # compile pipeline once finished adding modules, this step will form conf and dsl files for running job pipeline.compile() # fit model pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] backend = config.backend work_mode = config.work_mode guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) dataio_0 = DataIO(name="dataio_0") dataio_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, missing_fill=True, outlier_replace=True) dataio_0.get_party_instance(role='host', party_id=host).component_param(with_label=False, missing_fill=True, outlier_replace=True) intersection_0 = Intersection(name="intersection_0") federated_sample_0 = FederatedSample(name="federated_sample_0", mode="stratified", method="upsample", fractions=[[0, 1.5], [1, 2.0]]) feature_scale_0 = FeatureScale(name="feature_scale_0", method="min_max_scale", mode="normal") feature_scale_0.get_party_instance(role='guest', party_id=guest).component_param(feat_upper=[1, 2, 1, 1, 0.5, 1, 2, 2, 1, 1]) hetero_feature_binning_0 = HeteroFeatureBinning(name="hetero_feature_binning_0") hetero_feature_selection_0 = HeteroFeatureSelection(name="hetero_feature_selection_0") one_hot_0 = OneHotEncoder(name="one_hot_0") hetero_lr_0 = HeteroLR(name="hetero_lr_0", penalty="L2", optimizer="rmsprop", tol=1e-5, init_param={"init_method": "random_uniform"}, alpha=0.01, max_iter=10, early_stop="diff", batch_size=320, learning_rate=0.15) evaluation_0 = Evaluation(name="evaluation_0") pipeline.add_component(reader_0) pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data)) pipeline.add_component(federated_sample_0, data=Data(data=intersection_0.output.data)) pipeline.add_component(feature_scale_0, data=Data(data=federated_sample_0.output.data)) pipeline.add_component(hetero_feature_binning_0, data=Data(data=feature_scale_0.output.data)) pipeline.add_component(hetero_feature_selection_0, data=Data(data=hetero_feature_binning_0.output.data)) pipeline.add_component(one_hot_0, data=Data(data=hetero_feature_selection_0.output.data)) pipeline.add_component(hetero_lr_0, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component(evaluation_0, data=Data(data=hetero_lr_0.output.data)) pipeline.compile() job_parameters = JobParameters(backend=backend, work_mode=work_mode) pipeline.fit(job_parameters) print(pipeline.get_component("evaluation_0").get_summary())
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = {"name": "dvisits_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "dvisits_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0", with_label=False) intersection_0 = Intersection(name="intersection_0") feature_imputation_0 = FeatureImputation(name="feature_imputation_0", default_value=42, missing_impute=[0]) feature_imputation_0.get_party_instance(role='guest', party_id=guest).component_param( col_missing_fill_method={"doctorco": "min", "hscore": "designated"}) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(feature_imputation_0, data=Data(data=intersection_0.output.data)) pipeline.compile() pipeline.fit() # predict # deploy required components pipeline.deploy_component([data_transform_0, intersection_0, feature_imputation_0]) predict_pipeline = PipeLine() # add data reader onto predict pipeline predict_pipeline.add_component(reader_0) # add selected components from train pipeline onto predict pipeline # specify data source predict_pipeline.add_component(pipeline, data=Data(predict_input={pipeline.data_transform_0.input.data: reader_0.output.data})) # run predict model predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] backend = config.backend work_mode = config.work_mode guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "motor_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) dataio_0 = DataIO(name="dataio_0") dataio_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed", label_type="float", output_format="dense") dataio_0.get_party_instance(role='host', party_id=host).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0") hetero_data_split_0 = HeteroDataSplit(name="hetero_data_split_0", stratified=True, test_size=0.3, validate_size=0.2, split_points=[0.0, 0.2]) hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=10, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, encrypted_mode_calculator_param={"mode": "fast"}) hetero_linr_1 = HeteroLinR() pipeline.add_component(reader_0) pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data)) pipeline.add_component(hetero_data_split_0, data=Data(data=intersection_0.output.data)) pipeline.add_component(hetero_linr_0, data=Data(train_data=hetero_data_split_0.output.data.train_data, validate_data=hetero_data_split_0.output.data.validate_data)) pipeline.add_component(hetero_linr_1, data=Data(test_data=hetero_data_split_0.output.data.test_data), model=Model(model=hetero_linr_0.output.model)) pipeline.compile() job_parameters = JobParameters(backend=backend, work_mode=work_mode) pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] backend = config.backend work_mode = config.work_mode guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "tag_value_1000_140", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True) data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False, input_format="tag", tag_with_value=True) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.compile() job_parameters = JobParameters(backend=backend, work_mode=work_mode) pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] guest_train_data = {"name": "dvisits_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "dvisits_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="doctorco", label_type="float", output_format="dense", missing_fill=True, outlier_replace=False) data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False, output_format="dense", outlier_replace=False) intersection_0 = Intersection(name="intersection_0") hetero_poisson_0 = HeteroPoisson(name="hetero_poisson_0", early_stop="weight_diff", max_iter=10, optimizer="rmsprop", tol=0.001, penalty="L2", alpha=100.0, batch_size=-1, learning_rate=0.01, exposure_colname="exposure", decay_sqrt=False, init_param={"init_method": "zeros"}, encrypted_mode_calculator_param={"mode": "fast"}, cv_param={ "n_splits": 5, "shuffle": False, "random_seed": 103, "need_cv": True }) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(hetero_poisson_0, data=Data(train_data=intersection_0.output.data)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] backend = config.backend work_mode = config.work_mode guest_train_data = {"name": "breast_homo_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "breast_homo_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) dataio_0 = DataIO(name="dataio_0", with_label=True, output_format="dense", label_type="int", label_name="y") homo_lr_0 = HomoLR(name="homo_lr_0", penalty="L2", optimizer="sgd", tol=0.0001, alpha=0.01, max_iter=30, batch_size=-1, early_stop="weight_diff", learning_rate=0.15, init_param={"init_method": "zeros"}) local_baseline_0 = LocalBaseline(name="local_baseline_0", model_name="LogisticRegression", model_opts={"penalty": "l2", "tol": 0.0001, "C": 1.0, "fit_intercept": True, "solver": "saga", "max_iter": 2}) local_baseline_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True) local_baseline_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary", pos_label=1) evaluation_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True) evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) pipeline.add_component(reader_0) pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data)) pipeline.add_component(homo_lr_0, data=Data(train_data=dataio_0.output.data)) pipeline.add_component(local_baseline_0, data=Data(train_data=dataio_0.output.data)) pipeline.add_component(evaluation_0, data=Data(data=[homo_lr_0.output.data, local_baseline_0.output.data])) pipeline.compile() job_parameters = JobParameters(backend=backend, work_mode=work_mode) pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] guest_train_data = {"name": "vehicle_scale_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "vehicle_scale_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, output_format="dense", label_type="int", label_name="y") data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0", intersect_method="rsa", sync_intersect_ids=True, only_output_key=False) hetero_lr_0 = HeteroLR(name="hetero_lr_0", penalty="L2", optimizer="nesterov_momentum_sgd", tol=0.0001, alpha=0.0001, max_iter=30, batch_size=-1, early_stop="diff", learning_rate=0.15, init_param={"init_method": "zeros"}) local_baseline_0 = LocalBaseline(name="local_baseline_0", model_name="LogisticRegression", model_opts={"penalty": "l2", "tol": 0.0001, "C": 1.0, "fit_intercept": True, "solver": "lbfgs", "max_iter": 5, "multi_class": "ovr"}) local_baseline_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True) local_baseline_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) evaluation_0 = Evaluation(name="evaluation_0", eval_type="multi", pos_label=1) evaluation_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True) evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(hetero_lr_0, data=Data(train_data=intersection_0.output.data)) pipeline.add_component(local_baseline_0, data=Data(train_data=intersection_0.output.data)) pipeline.add_component(evaluation_0, data=Data(data=[hetero_lr_0.output.data, local_baseline_0.output.data])) pipeline.compile() pipeline.fit() # predict pipeline.deploy_component([data_transform_0, intersection_0, hetero_lr_0, local_baseline_0]) predict_pipeline = PipeLine() predict_pipeline.add_component(reader_0) predict_pipeline.add_component(pipeline, data=Data(predict_input={pipeline.data_transform_0.input.data: reader_0.output.data})) predict_pipeline.add_component(evaluation_0, data=Data(data=[hetero_lr_0.output.data, local_baseline_0.output.data])) predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] backend = config.backend work_mode = config.work_mode guest_train_data = {"name": "breast_hetero_mini_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "breast_hetero_mini_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).algorithm_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).algorithm_param(table=host_train_data) dataio_0 = DataIO(name="dataio_0") dataio_0.get_party_instance(role='guest', party_id=guest).algorithm_param(with_label=True, output_format="dense") dataio_0.get_party_instance(role='host', party_id=host).algorithm_param(with_label=False) intersection_0 = Intersection(name="intersection_0") hetero_lr_0 = HeteroLR(name="hetero_lr_0", early_stop="diff", max_iter=5, penalty="None", optimizer="sgd", tol=0.001, batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, encrypted_mode_calculator_param={"mode": "fast"}, stepwise_param={"score_name": "AIC", "direction": "backward", "need_stepwise": True, "max_step": 2, "nvmin": 2 }) pipeline.add_component(reader_0) pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data)) pipeline.add_component(hetero_lr_0, data=Data(train_data=intersection_0.output.data)) pipeline.compile() pipeline.fit(backend=backend, work_mode=work_mode)
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] backend = config.backend work_mode = config.work_mode guest_train_data = { "name": "ionosphere_scale_hetero_guest", "namespace": f"experiment{namespace}" } host_train_data = { "name": "ionosphere_scale_hetero_host", "namespace": f"experiment{namespace}" } pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance(role='guest', party_id=guest).component_param( with_label=True, label_name="LABEL", missing_fill=True, missing_fill_method="mean", outlier_replace=True) data_transform_0.get_party_instance(role='host', party_id=host).component_param( with_label=False, missing_fill=True, missing_fill_method="designated", default_value=0, outlier_replace=False) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.compile() job_parameters = JobParameters(backend=backend, work_mode=work_mode) pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] guest_train_data = [{"name": "tag_value_1", "namespace": f"experiment{namespace}"}, {"name": "tag_value_2", "namespace": f"experiment{namespace}"}, {"name": "tag_value_3", "namespace": f"experiment{namespace}"}] pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data[0]) reader_1 = Reader(name="reader_1") reader_1.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data[1]) reader_2 = Reader(name="reader_2") reader_2.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data[2]) union_0 = Union(name="union_0", allow_missing=False, keep_duplicate=True, need_run=True) data_transform_0 = DataTransform(name="data_transform_0", input_format="tag", with_label=False, tag_with_value=True, delimitor=",", output_format="dense") pipeline.add_component(reader_0) pipeline.add_component(reader_1) pipeline.add_component(reader_2) pipeline.add_component(union_0, data=Data(data=[reader_0.output.data, reader_1.output.data, reader_2.output.data])) pipeline.add_component(data_transform_0, data=Data(data=union_0.output.data)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = { "name": "breast_homo_guest", "namespace": f"experiment_sid{namespace}" } host_train_data = { "name": "breast_homo_host", "namespace": f"experiment_sid{namespace}" } pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0", with_match_id=True, with_label=True) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.compile() pipeline.fit()