def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"} host_train_data = {"name": "motor_hetero_host", "namespace": f"experiment{namespace}"} pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed", label_type="float", output_format="dense") data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0", only_output_key=False) hetero_linr_0 = HeteroSSHELinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=5, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, callback_param={"callbacks": ["ModelCheckpoint"]}, init_param={"init_method": "zeros"}) evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) hetero_linr_1 = HeteroSSHELinR(name="hetero_linr_1", max_iter=15, penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False ) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data)) pipeline.add_component(hetero_linr_1, data=Data(train_data=intersection_0.output.data), model=Model(hetero_linr_0.output.model)) pipeline.add_component(evaluation_0, data=Data(data=hetero_linr_1.output.data)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", param="./sshe_linr_config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] if isinstance(param, str): param = JobConfig.load_from_file(param) guest_train_data = { "name": "motor_hetero_guest", "namespace": f"experiment{namespace}" } host_train_data = { "name": "motor_hetero_host", "namespace": f"experiment{namespace}" } # initialize pipeline pipeline = PipeLine() # set job initiator pipeline.set_initiator(role='guest', party_id=guest) # set participants information pipeline.set_roles(guest=guest, host=host, arbiter=arbiter) # define Reader components to read in data reader_0 = Reader(name="reader_0") # configure Reader for guest reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) # configure Reader for host reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) # define DataTransform components data_transform_0 = DataTransform( name="data_transform_0") # start component numbering at 0 # get DataTransform party instance of guest data_transform_0_guest_party_instance = data_transform_0.get_party_instance( role='guest', party_id=guest) # configure DataTransform for guest data_transform_0_guest_party_instance.component_param( with_label=True, output_format="dense", label_name=param["label_name"], label_type="float") # get and configure DataTransform party instance of host data_transform_0.get_party_instance( role='host', party_id=host).component_param(with_label=False) # define Intersection component intersection_0 = Intersection(name="intersection_0") param = { "penalty": param["penalty"], "max_iter": param["max_iter"], "optimizer": param["optimizer"], "learning_rate": param["learning_rate"], "init_param": param["init_param"], "batch_size": param["batch_size"], "alpha": param["alpha"], "early_stop": param["early_stop"], "reveal_strategy": param["reveal_strategy"], "tol": 1e-6, "reveal_every_iter": True } hetero_sshe_linr_0 = HeteroSSHELinR(name='hetero_sshe_linr_0', **param) hetero_sshe_linr_1 = HeteroSSHELinR(name='hetero_sshe_linr_1') evaluation_0 = Evaluation(name='evaluation_0', eval_type="regression", metrics=[ "r2_score", "mean_squared_error", "root_mean_squared_error", "explained_variance" ]) # add components to pipeline, in order of task execution pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(hetero_sshe_linr_0, data=Data(train_data=intersection_0.output.data)) pipeline.add_component(hetero_sshe_linr_1, data=Data(test_data=intersection_0.output.data), model=Model(hetero_sshe_linr_0.output.model)) pipeline.add_component(evaluation_0, data=Data(data=hetero_sshe_linr_0.output.data)) # compile pipeline once finished adding modules, this step will form conf and dsl files for running job pipeline.compile() # fit model pipeline.fit() metric_summary = parse_summary_result( pipeline.get_component("evaluation_0").get_summary()) data_linr_0 = extract_data( pipeline.get_component("hetero_sshe_linr_0").get_output_data().get( "data"), "predict_result") data_linr_1 = extract_data( pipeline.get_component("hetero_sshe_linr_1").get_output_data().get( "data"), "predict_result") desc_linr_0 = regression_metric.Describe().compute(data_linr_0) desc_linr_1 = regression_metric.Describe().compute(data_linr_1) metric_summary["script_metrics"] = { "linr_train": desc_linr_0, "linr_validate": desc_linr_1 } data_summary = { "train": { "guest": guest_train_data["name"], "host": host_train_data["name"] }, "test": { "guest": guest_train_data["name"], "host": host_train_data["name"] } } return data_summary, metric_summary
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = [{ "name": "motor_hetero_guest", "namespace": f"experiment{namespace}" }, { "name": "motor_hetero_guest", "namespace": f"experiment{namespace}" }] host_train_data = [{ "name": "motor_hetero_host", "namespace": f"experiment{namespace}" }, { "name": "motor_hetero_host", "namespace": f"experiment{namespace}" }] pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data[0]) reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data[0]) reader_1 = Reader(name="reader_1") reader_1.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data[1]) reader_1.get_party_instance( role='host', party_id=host).component_param(table=host_train_data[1]) data_transform_0 = DataTransform(name="data_transform_0") data_transform_1 = DataTransform(name="data_transform_1") data_transform_0.get_party_instance( role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed", label_type="float", output_format="dense") data_transform_0.get_party_instance( role='host', party_id=host).component_param(with_label=False) data_transform_1.get_party_instance( role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed", label_type="float", output_format="dense") data_transform_1.get_party_instance( role='host', party_id=host).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0") intersect_1 = Intersection(name="intersection_1") hetero_linr_0 = HeteroSSHELinR( name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, callback_param={ "callbacks": ["EarlyStopping", "PerformanceEvaluate"], "validation_freqs": 1, "early_stopping_rounds": 5, "metrics": ["mean_absolute_error", "root_mean_squared_error"], "use_first_metric_only": False, "save_freq": 1 }, reveal_every_iter=True, reveal_strategy="respectively") pipeline.add_component(reader_0) pipeline.add_component(reader_1) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data), model=Model(data_transform_0.output.model)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(intersect_1, data=Data(data=data_transform_1.output.data)) pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data, validate_data=intersect_1.output.data)) pipeline.compile() pipeline.fit()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = { "name": "motor_hetero_guest", "namespace": f"experiment{namespace}" } host_train_data = { "name": "motor_hetero_host", "namespace": f"experiment{namespace}" } pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance( role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed", label_type="float", output_format="dense") data_transform_0.get_party_instance( role='host', party_id=host).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0") sample_weight_0 = SampleWeight(name="sample_weight_0") sample_weight_0.get_party_instance( role='guest', party_id=guest).component_param(need_run=True, sample_weight_name="pm") sample_weight_0.get_party_instance( role='host', party_id=host).component_param(need_run=False) hetero_linr_0 = HeteroSSHELinR(name="hetero_linr_0", penalty="L2", optimizer="rmsprop", tol=0.001, alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, reveal_every_iter=True, reveal_strategy="respectively") evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1) # evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(sample_weight_0, data=Data(data=intersection_0.output.data)) pipeline.add_component(hetero_linr_0, data=Data(train_data=sample_weight_0.output.data)) pipeline.add_component(evaluation_0, data=Data(data=hetero_linr_0.output.data)) pipeline.compile() pipeline.fit() # predict # deploy required components pipeline.deploy_component( [data_transform_0, intersection_0, hetero_linr_0]) predict_pipeline = PipeLine() # add data reader onto predict pipeline predict_pipeline.add_component(reader_0) # add selected components from train pipeline onto predict pipeline # specify data source predict_pipeline.add_component( pipeline, data=Data(predict_input={ pipeline.data_transform_0.input.data: reader_0.output.data })) # run predict model predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] arbiter = parties.arbiter[0] guest_train_data = { "name": "breast_hetero_guest", "namespace": f"experiment{namespace}" } host_train_data = { "name": "breast_hetero_host", "namespace": f"experiment{namespace}" } pipeline = PipeLine().set_initiator( role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) reader_1 = Reader(name="reader_1") reader_1.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) reader_1.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) reader_2 = Reader(name="reader_2") reader_2.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) reader_2.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance( role='guest', party_id=guest).component_param(with_label=True, missing_fill=True, outlier_replace=True) data_transform_0.get_party_instance( role='host', party_id=host).component_param(with_label=False, missing_fill=True, outlier_replace=True) data_transform_1 = DataTransform(name="data_transform_1") data_transform_2 = DataTransform(name="data_transform_2") intersection_0 = Intersection(name="intersection_0") intersection_1 = Intersection(name="intersection_1") intersection_2 = Intersection(name="intersection_2") union_0 = Union(name="union_0") federated_sample_0 = FederatedSample(name="federated_sample_0", mode="stratified", method="downsample", fractions=[[0, 1.0], [1, 1.0]]) feature_scale_0 = FeatureScale(name="feature_scale_0") feature_scale_1 = FeatureScale(name="feature_scale_1") hetero_feature_binning_0 = HeteroFeatureBinning( name="hetero_feature_binning_0") hetero_feature_binning_1 = HeteroFeatureBinning( name="hetero_feature_binning_1") hetero_feature_selection_0 = HeteroFeatureSelection( name="hetero_feature_selection_0") hetero_feature_selection_1 = HeteroFeatureSelection( name="hetero_feature_selection_1") one_hot_0 = OneHotEncoder(name="one_hot_0") one_hot_1 = OneHotEncoder(name="one_hot_1") hetero_lr_0 = HeteroLR(name="hetero_lr_0", penalty="L2", optimizer="rmsprop", tol=1e-5, init_param={"init_method": "random_uniform"}, alpha=0.01, max_iter=3, early_stop="diff", batch_size=320, learning_rate=0.15) hetero_lr_1 = HeteroLR(name="hetero_lr_1") hetero_lr_2 = HeteroLR(name="hetero_lr_2", penalty="L2", optimizer="rmsprop", tol=1e-5, init_param={"init_method": "random_uniform"}, alpha=0.01, max_iter=3, early_stop="diff", batch_size=320, learning_rate=0.15, cv_param={ "n_splits": 5, "shuffle": True, "random_seed": 103, "need_cv": True }) hetero_sshe_lr_0 = HeteroSSHELR( name="hetero_sshe_lr_0", reveal_every_iter=True, reveal_strategy="respectively", penalty="L2", optimizer="rmsprop", tol=1e-5, batch_size=320, learning_rate=0.15, init_param={"init_method": "random_uniform"}, alpha=0.01, max_iter=3) hetero_sshe_lr_1 = HeteroSSHELR(name="hetero_sshe_lr_1") local_baseline_0 = LocalBaseline(name="local_baseline_0", model_name="LogisticRegression", model_opts={ "penalty": "l2", "tol": 0.0001, "C": 1.0, "fit_intercept": True, "solver": "lbfgs", "max_iter": 5, "multi_class": "ovr" }) local_baseline_0.get_party_instance( role='guest', party_id=guest).component_param(need_run=True) local_baseline_0.get_party_instance( role='host', party_id=host).component_param(need_run=False) local_baseline_1 = LocalBaseline(name="local_baseline_1") hetero_secureboost_0 = HeteroSecureBoost(name="hetero_secureboost_0", num_trees=3) hetero_secureboost_1 = HeteroSecureBoost(name="hetero_secureboost_1") hetero_secureboost_2 = HeteroSecureBoost(name="hetero_secureboost_2", num_trees=3, cv_param={ "shuffle": False, "need_cv": True }) hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=3, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, floating_point_precision=23) hetero_linr_1 = HeteroLinR(name="hetero_linr_1") hetero_sshe_linr_0 = HeteroSSHELinR(name="hetero_sshe_linr_0", max_iter=5, early_stop="weight_diff", batch_size=-1) hetero_sshe_linr_1 = HeteroSSHELinR(name="hetero_sshe_linr_1") hetero_poisson_0 = HeteroPoisson(name="hetero_poisson_0", early_stop="weight_diff", max_iter=10, alpha=100.0, batch_size=-1, learning_rate=0.01, optimizer="rmsprop", exposure_colname="exposure", decay_sqrt=False, tol=0.001, init_param={"init_method": "zeros"}, penalty="L2") hetero_poisson_1 = HeteroPoisson(name="hetero_poisson_1") hetero_sshe_poisson_0 = HeteroSSHEPoisson(name="hetero_sshe_poisson_0", max_iter=5) hetero_sshe_poisson_1 = HeteroSSHEPoisson(name="hetero_sshe_poisson_1") evaluation_0 = Evaluation(name="evaluation_0") evaluation_1 = Evaluation(name="evaluation_1") evaluation_2 = Evaluation(name="evaluation_2") pipeline.add_component(reader_0) pipeline.add_component(reader_1) pipeline.add_component(reader_2) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data), model=Model(model=data_transform_0.output.model)) pipeline.add_component(data_transform_2, data=Data(data=reader_2.output.data), model=Model(model=data_transform_0.output.model)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(intersection_1, data=Data(data=data_transform_1.output.data)) pipeline.add_component(intersection_2, data=Data(data=data_transform_2.output.data)) pipeline.add_component( union_0, data=Data( data=[intersection_0.output.data, intersection_2.output.data])) pipeline.add_component(federated_sample_0, data=Data(data=intersection_1.output.data)) pipeline.add_component(feature_scale_0, data=Data(data=union_0.output.data)) pipeline.add_component(feature_scale_1, data=Data(data=federated_sample_0.output.data), model=Model(model=feature_scale_0.output.model)) pipeline.add_component(hetero_feature_binning_0, data=Data(data=feature_scale_0.output.data)) pipeline.add_component( hetero_feature_binning_1, data=Data(data=feature_scale_1.output.data), model=Model(model=hetero_feature_binning_0.output.model)) pipeline.add_component( hetero_feature_selection_0, data=Data(data=hetero_feature_binning_0.output.data)) pipeline.add_component( hetero_feature_selection_1, data=Data(data=hetero_feature_binning_1.output.data), model=Model(model=hetero_feature_selection_0.output.model)) pipeline.add_component( one_hot_0, data=Data(data=hetero_feature_selection_0.output.data)) pipeline.add_component( one_hot_1, data=Data(data=hetero_feature_selection_1.output.data), model=Model(model=one_hot_0.output.model)) pipeline.add_component(hetero_lr_0, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component(hetero_lr_1, data=Data(test_data=one_hot_1.output.data), model=Model(model=hetero_lr_0.output.model)) pipeline.add_component(hetero_lr_2, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component(local_baseline_0, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component(local_baseline_1, data=Data(test_data=one_hot_1.output.data), model=Model(model=local_baseline_0.output.model)) pipeline.add_component(hetero_sshe_lr_0, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component(hetero_sshe_lr_1, data=Data(test_data=one_hot_1.output.data), model=Model(model=hetero_sshe_lr_0.output.model)) pipeline.add_component(hetero_secureboost_0, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component( hetero_secureboost_1, data=Data(test_data=one_hot_1.output.data), model=Model(model=hetero_secureboost_0.output.model)) pipeline.add_component(hetero_secureboost_2, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component(hetero_linr_0, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component(hetero_linr_1, data=Data(test_data=one_hot_1.output.data), model=Model(model=hetero_linr_0.output.model)) pipeline.add_component(hetero_sshe_linr_0, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component(hetero_sshe_linr_1, data=Data(test_data=one_hot_1.output.data), model=Model(model=hetero_sshe_linr_0.output.model)) pipeline.add_component(hetero_poisson_0, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component(hetero_poisson_1, data=Data(test_data=one_hot_1.output.data), model=Model(model=hetero_poisson_0.output.model)) pipeline.add_component( evaluation_0, data=Data(data=[ hetero_lr_0.output.data, hetero_lr_1.output.data, hetero_sshe_lr_0.output.data, hetero_sshe_lr_1.output.data, local_baseline_0.output.data, local_baseline_1.output.data ])) pipeline.add_component(hetero_sshe_poisson_0, data=Data(train_data=one_hot_0.output.data)) pipeline.add_component( hetero_sshe_poisson_1, data=Data(test_data=one_hot_1.output.data), model=Model(model=hetero_sshe_poisson_0.output.model)) pipeline.add_component( evaluation_1, data=Data(data=[ hetero_linr_0.output.data, hetero_linr_1.output.data, hetero_sshe_linr_0.output.data, hetero_linr_1.output.data ])) pipeline.add_component( evaluation_2, data=Data(data=[ hetero_poisson_0.output.data, hetero_poisson_1.output.data, hetero_sshe_poisson_0.output.data, hetero_sshe_poisson_1.output.data ])) pipeline.compile() pipeline.fit() print(pipeline.get_component("evaluation_0").get_summary()) print(pipeline.get_component("evaluation_1").get_summary()) print(pipeline.get_component("evaluation_2").get_summary())
def main(config="../../config.yaml", namespace=""): # obtain config if isinstance(config, str): config = load_job_config(config) parties = config.parties guest = parties.guest[0] host = parties.host[0] guest_train_data = { "name": "motor_hetero_guest", "namespace": f"experiment{namespace}" } host_train_data = { "name": "motor_hetero_host", "namespace": f"experiment{namespace}" } pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host) reader_0 = Reader(name="reader_0") reader_0.get_party_instance( role='guest', party_id=guest).component_param(table=guest_train_data) reader_0.get_party_instance( role='host', party_id=host).component_param(table=host_train_data) data_transform_0 = DataTransform(name="data_transform_0") data_transform_0.get_party_instance( role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed", label_type="float", output_format="dense") data_transform_0.get_party_instance( role='host', party_id=host).component_param(with_label=False) intersection_0 = Intersection(name="intersection_0") hetero_linr_0 = HeteroSSHELinR(name="hetero_linr_0", penalty="None", optimizer="sgd", tol=0.001, alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1, learning_rate=0.15, decay=0.0, decay_sqrt=False, init_param={"init_method": "zeros"}, cv_param={ "n_splits": 5, "shuffle": False, "random_seed": 42, "need_cv": True }) pipeline.add_component(reader_0) pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data)) pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data)) pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data)) pipeline.compile() pipeline.fit()