def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role="guest", party_id=guest).set_roles(guest=guest)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(
role="guest", party_id=guest).component_param(table=guest_train_data)
# define ColumnExpand components
column_expand_0 = ColumnExpand(name="column_expand_0")
column_expand_0.get_party_instance(
role="guest", party_id=guest).component_param(
need_run=True,
method="manual",
append_header=["x_0", "x_1", "x_2", "x_3"],
fill_value=[0, 0.2, 0.5, 1])
# define DataIO components
dataio_0 = DataIO(name="dataio_0") # start component numbering at 0
# get DataIO party instance of guest
dataio_0_guest_party_instance = dataio_0.get_party_instance(role="guest",
party_id=guest)
# configure DataIO for guest
dataio_0_guest_party_instance.component_param(with_label=True,
output_format="dense")
# add components to pipeline, in order of task execution
pipeline.add_component(reader_0)
pipeline.add_component(column_expand_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(dataio_0,
data=Data(data=column_expand_0.output.data))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
job_parameters = JobParameters(backend=backend, work_mode=work_mode)
pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
guest_train_data = [{"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"},
{"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"}]
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data[0])
reader_1 = Reader(name="reader_1")
reader_1.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data[1])
union_0 = Union(name="union_0", allow_missing=False, keep_duplicate=True)
data_transform_0 = DataTransform(name="data_transform_0", with_label=True, output_format="dense", label_name="y",
missing_fill=False, outlier_replace=False)
pipeline.add_component(reader_0)
pipeline.add_component(reader_1)
pipeline.add_component(union_0, data=Data(data=[reader_0.output.data, reader_1.output.data]))
pipeline.add_component(data_transform_0, data=Data(data=union_0.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).algorithm_param(table=guest_train_data)
reader_1 = Reader(name="reader_1")
reader_1.get_party_instance(
role='guest', party_id=guest).algorithm_param(table=guest_train_data)
dataio_0 = DataIO(name="dataio_0",
with_label=True,
output_format="dense",
label_name="y",
missing_fill=False,
outlier_replace=False)
dataio_1 = DataIO(name="dataio_1",
with_label=True,
output_format="dense",
label_name="y",
missing_fill=False,
outlier_replace=False)
union_0 = Union(name="union_0", allow_missing=False, need_run=True)
pipeline.add_component(reader_0)
pipeline.add_component(reader_1)
pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data))
pipeline.add_component(dataio_1,
data=Data(data=reader_1.output.data),
model=Model(dataio_0.output.model))
pipeline.add_component(
union_0, data=Data(data=[dataio_0.output.data, dataio_1.output.data]))
pipeline.compile()
pipeline.fit(backend=backend, work_mode=work_mode)
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
hosts = parties.host
arbiter = parties.arbiter[0]
guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = [{"name": "motor_hetero_host", "namespace": f"experiment{namespace}"},
{"name": "motor_hetero_host", "namespace": f"experiment{namespace}"}]
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=hosts, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=hosts[0]).component_param(table=host_train_data[0])
reader_0.get_party_instance(role='host', party_id=hosts[1]).component_param(table=host_train_data[1])
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed",
label_type="float", output_format="dense")
data_transform_0.get_party_instance(role='host', party_id=hosts).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001,
alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1,
learning_rate=0.15, decay=0.0, decay_sqrt=False,
init_param={"init_method": "zeros"},
encrypted_mode_calculator_param={"mode": "fast"})
evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1)
# evaluation_0.get_party_instance(role='host', party_id=hosts[0]).component_param(need_run=False)
# evaluation_0.get_party_instance(role='host', party_id=hosts[1]).component_param(need_run=False)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data))
pipeline.add_component(evaluation_0, data=Data(data=hetero_linr_0.output.data))
pipeline.compile()
pipeline.fit()
# predict
# deploy required components
pipeline.deploy_component([data_transform_0, intersection_0, hetero_linr_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_0)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(pipeline,
data=Data(predict_input={pipeline.data_transform_0.input.data: reader_0.output.data}))
# run predict model
predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment_sid{namespace}"}
host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment_sid{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0", with_label=False, with_match_id=True)
intersection_0 = Intersection(name="intersection_0")
feature_imputation_0 = FeatureImputation(name="feature_imputation_0",
missing_fill_method="designated",
default_value=42, missing_impute=[0])
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(feature_imputation_0, data=Data(data=intersection_0.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {"name": "vehicle_scale_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "vehicle_scale_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, output_format="dense",
label_type="int", label_name="y")
data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0", intersect_method="rsa", sync_intersect_ids=True,
only_output_key=False)
hetero_lr_0 = HeteroLR(name="hetero_lr_0", penalty="L2", optimizer="nesterov_momentum_sgd",
tol=0.0001, alpha=0.0001, max_iter=30, batch_size=-1,
early_stop="diff", learning_rate=0.15, init_param={"init_method": "zeros"})
local_baseline_0 = LocalBaseline(name="local_baseline_0", model_name="LogisticRegression",
model_opts={"penalty": "l2", "tol": 0.0001, "C": 1.0, "fit_intercept": True,
"solver": "lbfgs", "max_iter": 5, "multi_class": "ovr"})
local_baseline_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True)
local_baseline_0.get_party_instance(role='host', party_id=host).component_param(need_run=False)
evaluation_0 = Evaluation(name="evaluation_0", eval_type="multi", pos_label=1)
evaluation_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True)
evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_lr_0, data=Data(train_data=intersection_0.output.data))
pipeline.add_component(local_baseline_0, data=Data(train_data=intersection_0.output.data))
pipeline.add_component(evaluation_0, data=Data(data=[hetero_lr_0.output.data, local_baseline_0.output.data]))
pipeline.compile()
pipeline.fit()
# predict
pipeline.deploy_component([data_transform_0, intersection_0, hetero_lr_0, local_baseline_0])
predict_pipeline = PipeLine()
predict_pipeline.add_component(reader_0)
predict_pipeline.add_component(pipeline,
data=Data(predict_input={pipeline.data_transform_0.input.data: reader_0.output.data}))
predict_pipeline.add_component(evaluation_0, data=Data(data=[hetero_lr_0.output.data, local_baseline_0.output.data]))
predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "tag_value_1000_140", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True)
data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False,
input_format="tag",
tag_with_value=True)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.compile()
job_parameters = JobParameters(backend=backend, work_mode=work_mode)
pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role="guest", party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0") # start component numbering at 0
data_transform_0_guest_party_instance = data_transform_0.get_party_instance(role="guest", party_id=guest)
data_transform_0_guest_party_instance.component_param(with_label=True, output_format="dense")
data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False,
output_format="dense")
intersection_0 = Intersection(name="intersection_0")
label_transform_0 = LabelTransform(name="label_transform_0")
label_transform_0.get_party_instance(role="host", party_id=host).component_param(need_run=False)
hetero_lr_0 = HeteroLR(name="hetero_lr_0", penalty="L2", optimizer="sgd", tol=0.001,
alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1,
learning_rate=0.15, decay=0.0, decay_sqrt=False,
init_param={"init_method": "zeros"},
encrypted_mode_calculator_param={"mode": "fast"},
floating_point_precision=23)
label_transform_1 = LabelTransform(name="label_transform_1")
evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary", pos_label=1)
# add components to pipeline, in order of task execution
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(label_transform_0, data=Data(data=intersection_0.output.data))
pipeline.add_component(hetero_lr_0, data=Data(train_data=label_transform_0.output.data))
pipeline.add_component(label_transform_1, data=Data(data=hetero_lr_0.output.data), model=Model(label_transform_0.output.model))
pipeline.add_component(evaluation_0, data=Data(data=label_transform_1.output.data))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "breast_homo_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_homo_host",
"namespace": f"experiment{namespace}"
}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role='guest', party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=host, arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
# define DataTransform components
data_transform_0 = DataTransform(
name="data_transform_0", with_label=True,
output_format="dense") # start component numbering at 0
homo_binning_0 = HomoFeatureBinning(name='homo_binning_0',
sample_bins=1000)
homo_binning_1 = HomoFeatureBinning(name='homo_binning_1',
sample_bins=1000)
# add components to pipeline, in order of task execution
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
# set data input sources of intersection components
pipeline.add_component(homo_binning_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(homo_binning_1,
data=Data(data=data_transform_0.output.data),
model=Model(model=homo_binning_0.output.model))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True)
data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_nn_0 = HeteroNN(name="hetero_nn_0", epochs=100,
interactive_layer_lr=0.15, batch_size=-1, early_stop="diff",
selector_param={"method": "relative"})
guest_nn_0 = hetero_nn_0.get_party_instance(role='guest', party_id=guest)
guest_nn_0.add_bottom_model(Dense(units=3, input_shape=(10,), activation="relu",
kernel_initializer=initializers.Constant(value=1)))
guest_nn_0.set_interactve_layer(Dense(units=2, input_shape=(2,),
kernel_initializer=initializers.Constant(value=1)))
guest_nn_0.add_top_model(Dense(units=1, input_shape=(2,), activation="sigmoid",
kernel_initializer=initializers.Constant(value=1)))
host_nn_0 = hetero_nn_0.get_party_instance(role='host', party_id=host)
host_nn_0.add_bottom_model(Dense(units=3, input_shape=(20,), activation="relu",
kernel_initializer=initializers.Constant(value=1)))
host_nn_0.set_interactve_layer(Dense(units=2, input_shape=(2,),
kernel_initializer=initializers.Constant(value=1)))
hetero_nn_0.compile(optimizer=optimizers.SGD(lr=0.15), loss="binary_crossentropy")
hetero_nn_1 = HeteroNN(name="hetero_nn_1")
evaluation_0 = Evaluation(name="evaluation_0")
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_nn_0, data=Data(train_data=intersection_0.output.data))
pipeline.add_component(hetero_nn_1, data=Data(test_data=intersection_0.output.data),
model=Model(model=hetero_nn_0.output.model))
pipeline.add_component(evaluation_0, data=Data(data=hetero_nn_0.output.data))
pipeline.compile()
pipeline.fit()
print(hetero_nn_0.get_config(roles={"guest": [guest],
"host": [host]}))
print(pipeline.get_component("hetero_nn_0").get_summary())
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {"name": "dvisits_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "dvisits_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).algorithm_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).algorithm_param(table=host_train_data)
dataio_0 = DataIO(name="dataio_0")
dataio_0.get_party_instance(role='guest', party_id=guest).algorithm_param(with_label=True, label_name="doctorco",
label_type="float", output_format="dense")
dataio_0.get_party_instance(role='host', party_id=host).algorithm_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_poisson_0 = HeteroPoisson(name="hetero_poisson_0", early_stop="weight_diff", max_iter=10,
alpha=100.0, batch_size=-1, learning_rate=0.01, optimizer="rmsprop",
exposure_colname="exposure", decay_sqrt=False, tol=0.001,
init_param={"init_method": "zeros"}, penalty="L2",
encrypted_mode_calculator_param={"mode": "fast"})
evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1)
evaluation_0.get_party_instance(role='host', party_id=host).algorithm_param(need_run=False)
pipeline.add_component(reader_0)
pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data))
pipeline.add_component(hetero_poisson_0, data=Data(train_data=intersection_0.output.data))
pipeline.add_component(evaluation_0, data=Data(data=hetero_poisson_0.output.data))
pipeline.compile()
pipeline.fit(backend=backend, work_mode=work_mode)
# predict
# deploy required components
pipeline.deploy_component([dataio_0, intersection_0, hetero_poisson_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_0)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(pipeline,
data=Data(predict_input={pipeline.dataio_0.input.data: reader_0.output.data}))
# run predict model
predict_pipeline.predict(backend=backend, work_mode=work_mode)
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=False,
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False,
output_format="dense")
param = {
"intersect_method": "raw",
"sync_intersect_ids": True,
"only_output_key": True,
"raw_params": {
"use_hash": True,
"hash_method": "sm3",
"salt": "12345",
"base64": False,
"join_role": "host"
}
}
intersect_0 = Intersection(name="intersect_0", **param)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersect_0,
data=Data(data=data_transform_0.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(
with_label=False, output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(
with_label=False, output_format="dense")
param_0 = {
"intersect_method": "rsa",
"rsa_params": {
"hash_method": "sha256",
"final_hash_method": "sha256",
"key_length": 2048
},
"run_cache": True
}
param_1 = {
"intersect_method": "rsa",
"sync_intersect_ids": False,
"only_output_key": True,
"rsa_params": {
"hash_method": "sha256",
"final_hash_method": "sha256",
"key_length": 2048
}
}
intersect_0 = Intersection(name="intersect_0", **param_0)
intersect_1 = Intersection(name="intersect_1", **param_1)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(
intersect_1, data=Data(
data=data_transform_0.output.data), cache=Cache(
intersect_0.output.cache))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
# data sets
guest_train_data = {"name": "student_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "student_hetero_host", "namespace": f"experiment{namespace}"}
# init pipeline
pipeline = PipeLine().set_initiator(role="guest", party_id=guest).set_roles(guest=guest, host=host,)
# set data reader and data-io
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role="guest",
party_id=guest).component_param(
with_label=True,
output_format="dense",
label_type="float")
data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False)
# data intersect component
intersect_0 = Intersection(name="intersection_0")
# secure boost component
hetero_secure_boost_0 = HeteroSecureBoost(name="hetero_secure_boost_0",
num_trees=3,
task_type="regression",
objective_param={"objective": "lse"},
encrypt_param={"method": "Paillier"},
tree_param={"max_depth": 3},
validation_freqs=1,
cv_param={
"need_cv": True,
"n_splits": 5,
"shuffle": False,
"random_seed": 103
}
)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_secure_boost_0, data=Data(train_data=intersect_0.output.data))
pipeline.compile()
pipeline.fit()
print("fitting hetero secureboost done, result:")
print(pipeline.get_component("hetero_secure_boost_0").get_summary())
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "motor_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest',
party_id=guest).component_param(
with_label=True,
label_name="motor_speed",
label_type="float",
output_format="dense")
data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0", only_output_key=False)
hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001,
alpha=0.01, max_iter=5, early_stop="weight_diff", batch_size=-1,
learning_rate=0.15, decay=0.0, decay_sqrt=False,
callback_param={"callbacks": ["ModelCheckpoint"]},
init_param={"init_method": "zeros"},
floating_point_precision=23)
evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression", pos_label=1)
hetero_linr_1 = HeteroLinR(name="hetero_linr_1", max_iter=15,
penalty="L2", optimizer="sgd", tol=0.001,
alpha=0.01, early_stop="weight_diff", batch_size=-1,
learning_rate=0.15, decay=0.0, decay_sqrt=False,
floating_point_precision=23
)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data))
pipeline.add_component(hetero_linr_1, data=Data(train_data=intersection_0.output.data),
model=Model(hetero_linr_0.output.model))
pipeline.add_component(evaluation_0, data=Data(data=hetero_linr_1.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
dataio_0 = DataIO(name="dataio_0")
dataio_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, missing_fill=True,
outlier_replace=True)
dataio_0.get_party_instance(role='host', party_id=host).component_param(with_label=False, missing_fill=True,
outlier_replace=True)
intersection_0 = Intersection(name="intersection_0")
federated_sample_0 = FederatedSample(name="federated_sample_0", mode="stratified", method="upsample",
fractions=[[0, 1.5], [1, 2.0]])
feature_scale_0 = FeatureScale(name="feature_scale_0", method="min_max_scale", mode="normal")
feature_scale_0.get_party_instance(role='guest', party_id=guest).component_param(feat_upper=[1, 2, 1, 1, 0.5, 1, 2, 2, 1, 1])
hetero_feature_binning_0 = HeteroFeatureBinning(name="hetero_feature_binning_0")
hetero_feature_selection_0 = HeteroFeatureSelection(name="hetero_feature_selection_0")
one_hot_0 = OneHotEncoder(name="one_hot_0")
hetero_lr_0 = HeteroLR(name="hetero_lr_0", penalty="L2", optimizer="rmsprop", tol=1e-5,
init_param={"init_method": "random_uniform"},
alpha=0.01, max_iter=10, early_stop="diff", batch_size=320, learning_rate=0.15)
evaluation_0 = Evaluation(name="evaluation_0")
pipeline.add_component(reader_0)
pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data))
pipeline.add_component(federated_sample_0, data=Data(data=intersection_0.output.data))
pipeline.add_component(feature_scale_0, data=Data(data=federated_sample_0.output.data))
pipeline.add_component(hetero_feature_binning_0, data=Data(data=feature_scale_0.output.data))
pipeline.add_component(hetero_feature_selection_0, data=Data(data=hetero_feature_binning_0.output.data))
pipeline.add_component(one_hot_0, data=Data(data=hetero_feature_selection_0.output.data))
pipeline.add_component(hetero_lr_0, data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(evaluation_0, data=Data(data=hetero_lr_0.output.data))
pipeline.compile()
job_parameters = JobParameters(backend=backend, work_mode=work_mode)
pipeline.fit(job_parameters)
print(pipeline.get_component("evaluation_0").get_summary())
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {
"name": "breast_homo_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_homo_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
dataio_0 = DataIO(name="dataio_0")
dataio_0.get_party_instance(role='guest', party_id=guest).component_param(
with_label=True,
output_format="dense",
label_name="y",
label_type="int")
dataio_0.get_party_instance(role='host',
party_id=host).component_param(with_label=True)
homo_data_split_0 = HomoDataSplit(name="homo_data_split_0",
stratified=True,
test_size=0.3,
validate_size=0.2)
pipeline.add_component(reader_0)
pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data))
pipeline.add_component(homo_data_split_0,
data=Data(data=dataio_0.output.data))
pipeline.compile()
job_parameters = JobParameters(backend=backend, work_mode=work_mode)
pipeline.fit(job_parameters)
print(pipeline.get_component("homo_data_split_0").get_summary())
def run_homo_nn_pipeline(config, namespace, data: dict, nn_component,
num_host):
if isinstance(config, str):
config = load_job_config(config)
guest_train_data = data["guest"]
host_train_data = data["host"][:num_host]
for d in [guest_train_data, *host_train_data]:
d["namespace"] = f"{d['namespace']}{namespace}"
hosts = config.parties.host[:num_host]
pipeline = (PipeLine().set_initiator(
role="guest", party_id=config.parties.guest[0]).set_roles(
guest=config.parties.guest[0],
host=hosts,
arbiter=config.parties.arbiter))
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role="guest", party_id=config.parties.guest[0]).component_param(
table=guest_train_data)
for i in range(num_host):
reader_0.get_party_instance(
role="host",
party_id=hosts[i]).component_param(table=host_train_data[i])
data_transform_0 = DataTransform(name="data_transform_0", with_label=True)
data_transform_0.get_party_instance(
role="guest", party_id=config.parties.guest[0]).component_param(
with_label=True, output_format="dense")
data_transform_0.get_party_instance(
role="host", party_id=hosts).component_param(with_label=True)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(nn_component,
data=Data(train_data=data_transform_0.output.data))
pipeline.compile()
pipeline.fit()
print(pipeline.get_component("homo_nn_0").get_summary())
pipeline.deploy_component([data_transform_0, nn_component])
# predict
predict_pipeline = PipeLine()
predict_pipeline.add_component(reader_0)
predict_pipeline.add_component(
pipeline,
data=Data(predict_input={
pipeline.data_transform_0.input.data: reader_0.output.data
}),
)
# run predict model
predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {
"name": "ionosphere_scale_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "ionosphere_scale_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(role='guest',
party_id=guest).component_param(
with_label=True,
label_name="LABEL",
missing_fill=True,
missing_fill_method="mean",
outlier_replace=True)
data_transform_0.get_party_instance(role='host',
party_id=host).component_param(
with_label=False,
missing_fill=True,
missing_fill_method="designated",
default_value=0,
outlier_replace=False)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.compile()
job_parameters = JobParameters(backend=backend, work_mode=work_mode)
pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
hosts = parties.host
arbiter = parties.arbiter[0]
guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = [{"name": "motor_hetero_host", "namespace": f"experiment{namespace}"},
{"name": "motor_hetero_host", "namespace": f"experiment{namespace}"}]
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(
guest=guest, host=hosts, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=hosts[0]).component_param(table=host_train_data[0])
reader_0.get_party_instance(role='host', party_id=hosts[1]).component_param(table=host_train_data[1])
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest',
party_id=guest).component_param(
with_label=True,
label_name="motor_speed",
label_type="float",
output_format="dense")
data_transform_0.get_party_instance(role='host', party_id=hosts).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="None", optimizer="sgd", tol=0.001,
alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1,
learning_rate=0.15, decay=0.0, decay_sqrt=False,
init_param={"init_method": "zeros"},
cv_param={"n_splits": 5,
"shuffle": False,
"random_seed": 42,
"need_cv": True
}
)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
hosts = parties.host
arbiter = parties.arbiter[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {"name": "breast_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "breast_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=hosts, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=hosts).component_param(table=host_train_data)
dataio_0 = DataIO(name="dataio_0")
dataio_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="y",
label_type="int", output_format="dense")
dataio_0.get_party_instance(role='host', party_id=hosts).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
sample_weight_0 = SampleWeight(name="sample_weight_0")
sample_weight_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True,
class_weight="balanced")
sample_weight_0.get_party_instance(role='host', party_id=hosts).component_param(need_run=False)
hetero_lr_0 = HeteroLR(name="hetero_lr_0", optimizer="nesterov_momentum_sgd", tol=0.001,
alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1,
learning_rate=0.15,
init_param={"init_method": "zeros"})
evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary", pos_label=1)
# evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False)
pipeline.add_component(reader_0)
pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data))
pipeline.add_component(sample_weight_0, data=Data(data=intersection_0.output.data))
pipeline.add_component(hetero_lr_0, data=Data(train_data=sample_weight_0.output.data))
pipeline.add_component(evaluation_0, data=Data(data=hetero_lr_0.output.data))
pipeline.compile()
job_parameters = JobParameters(backend=backend, work_mode=work_mode)
pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
hosts = parties.host
guest_train_data = {"name": "breast_homo_test", "namespace": f"experiment_sid{namespace}"}
host_train_data = {"name": "breast_homo_test", "namespace": f"experiment_sid{namespace}"}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role="guest", party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=hosts)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(role="host", party_id=hosts).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0", with_match_id=True)
# get and configure DataTransform party instance of guest
data_transform_0.get_party_instance(
role="guest", party_id=guest).component_param(
with_label=False, output_format="dense")
# get and configure DataTransform party instance of host
data_transform_0.get_party_instance(role="host", party_id=hosts).component_param(with_label=False)
# define FeldmanVerifiableSum components
feldmanverifiablesum_0 = FeldmanVerifiableSum(name="feldmanverifiablesum_0")
feldmanverifiablesum_0.get_party_instance(role="guest", party_id=guest).component_param(sum_cols=[1, 2, 3], q_n=6)
feldmanverifiablesum_0.get_party_instance(role="host", party_id=hosts).component_param(sum_cols=[1, 2, 3], q_n=6)
# add components to pipeline, in order of task execution.
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(feldmanverifiablesum_0, data=Data(data=data_transform_0.output.data))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {"name": "dvisits_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "dvisits_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0", with_label=False)
intersection_0 = Intersection(name="intersection_0")
feature_imputation_0 = FeatureImputation(name="feature_imputation_0",
default_value=42,
missing_impute=[0])
feature_imputation_0.get_party_instance(role='guest', party_id=guest).component_param(
col_missing_fill_method={"doctorco": "min",
"hscore": "designated"})
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(feature_imputation_0, data=Data(data=intersection_0.output.data))
pipeline.compile()
pipeline.fit()
# predict
# deploy required components
pipeline.deploy_component([data_transform_0, intersection_0,
feature_imputation_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_0)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(pipeline,
data=Data(predict_input={pipeline.data_transform_0.input.data: reader_0.output.data}))
# run predict model
predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "motor_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
dataio_0 = DataIO(name="dataio_0")
dataio_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed",
label_type="float", output_format="dense")
dataio_0.get_party_instance(role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_data_split_0 = HeteroDataSplit(name="hetero_data_split_0", stratified=True,
test_size=0.3, validate_size=0.2, split_points=[0.0, 0.2])
hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001,
alpha=0.01, max_iter=10, early_stop="weight_diff", batch_size=-1,
learning_rate=0.15, decay=0.0, decay_sqrt=False,
init_param={"init_method": "zeros"},
encrypted_mode_calculator_param={"mode": "fast"})
hetero_linr_1 = HeteroLinR()
pipeline.add_component(reader_0)
pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data))
pipeline.add_component(hetero_data_split_0, data=Data(data=intersection_0.output.data))
pipeline.add_component(hetero_linr_0, data=Data(train_data=hetero_data_split_0.output.data.train_data,
validate_data=hetero_data_split_0.output.data.validate_data))
pipeline.add_component(hetero_linr_1, data=Data(test_data=hetero_data_split_0.output.data.test_data),
model=Model(model=hetero_linr_0.output.model))
pipeline.compile()
job_parameters = JobParameters(backend=backend, work_mode=work_mode)
pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {"name": "dvisits_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "dvisits_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="doctorco",
label_type="float", output_format="dense",
missing_fill=True, outlier_replace=False)
data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False, output_format="dense",
outlier_replace=False)
intersection_0 = Intersection(name="intersection_0")
hetero_poisson_0 = HeteroPoisson(name="hetero_poisson_0", early_stop="weight_diff", max_iter=10,
optimizer="rmsprop", tol=0.001, penalty="L2",
alpha=100.0, batch_size=-1, learning_rate=0.01,
exposure_colname="exposure", decay_sqrt=False,
init_param={"init_method": "zeros"},
encrypted_mode_calculator_param={"mode": "fast"},
cv_param={
"n_splits": 5,
"shuffle": False,
"random_seed": 103,
"need_cv": True
})
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_poisson_0, data=Data(train_data=intersection_0.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {
"name": "svmlight_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "svmlight_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0",
input_format="sparse")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True)
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
data_transform_1 = DataTransform(name="data_transform_1")
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(data_transform_1,
data=Data(data=reader_0.output.data),
model=Model(model=data_transform_0.output.model))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {"name": "breast_homo_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "breast_homo_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
dataio_0 = DataIO(name="dataio_0", with_label=True, output_format="dense",
label_type="int", label_name="y")
homo_lr_0 = HomoLR(name="homo_lr_0", penalty="L2", optimizer="sgd",
tol=0.0001, alpha=0.01, max_iter=30, batch_size=-1,
early_stop="weight_diff", learning_rate=0.15, init_param={"init_method": "zeros"})
local_baseline_0 = LocalBaseline(name="local_baseline_0", model_name="LogisticRegression",
model_opts={"penalty": "l2", "tol": 0.0001, "C": 1.0, "fit_intercept": True,
"solver": "saga", "max_iter": 2})
local_baseline_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True)
local_baseline_0.get_party_instance(role='host', party_id=host).component_param(need_run=False)
evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary", pos_label=1)
evaluation_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True)
evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False)
pipeline.add_component(reader_0)
pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data))
pipeline.add_component(homo_lr_0, data=Data(train_data=dataio_0.output.data))
pipeline.add_component(local_baseline_0, data=Data(train_data=dataio_0.output.data))
pipeline.add_component(evaluation_0, data=Data(data=[homo_lr_0.output.data, local_baseline_0.output.data]))
pipeline.compile()
job_parameters = JobParameters(backend=backend, work_mode=work_mode)
pipeline.fit(job_parameters)
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
backend = config.backend
work_mode = config.work_mode
guest_train_data = {"name": "breast_hetero_mini_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "breast_hetero_mini_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).algorithm_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).algorithm_param(table=host_train_data)
dataio_0 = DataIO(name="dataio_0")
dataio_0.get_party_instance(role='guest', party_id=guest).algorithm_param(with_label=True, output_format="dense")
dataio_0.get_party_instance(role='host', party_id=host).algorithm_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_lr_0 = HeteroLR(name="hetero_lr_0", early_stop="diff", max_iter=5,
penalty="None", optimizer="sgd", tol=0.001,
batch_size=-1, learning_rate=0.15, decay=0.0,
decay_sqrt=False,
init_param={"init_method": "zeros"},
encrypted_mode_calculator_param={"mode": "fast"},
stepwise_param={"score_name": "AIC", "direction": "backward",
"need_stepwise": True, "max_step": 2, "nvmin": 2
})
pipeline.add_component(reader_0)
pipeline.add_component(dataio_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=dataio_0.output.data))
pipeline.add_component(hetero_lr_0, data=Data(train_data=intersection_0.output.data))
pipeline.compile()
pipeline.fit(backend=backend, work_mode=work_mode)
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {
"name": "breast_homo_guest",
"namespace": f"experiment_sid{namespace}"
}
host_train_data = {
"name": "breast_homo_host",
"namespace": f"experiment_sid{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0",
with_match_id=True,
with_label=True)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {"name": "breast_homo_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "breast_homo_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
reader_1 = Reader(name="reader_1")
reader_1.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_1.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0", with_label=True)
data_transform_1 = DataTransform(name="data_transform_1")
federated_sample_0 = FederatedSample(name="federated_sample_0", mode="stratified", method="downsample",
fractions=[[0, 1.0], [1, 1.0]], task_type="h**o")
homo_binning_0 = HomoFeatureBinning(name='homo_binning_0', sample_bins=10, method="recursive_query")
homo_binning_1 = HomoFeatureBinning(name='homo_binning_1')
homo_onehot_0 = HomoOneHotEncoder(name='homo_onehot_0', need_alignment=True)
homo_onehot_1 = HomoOneHotEncoder(name='homo_onehot_1')
homo_lr_0 = HomoLR(name="homo_lr_0", penalty="L2", tol=0.0001, alpha=1.0,
optimizer="rmsprop", max_iter=5)
homo_lr_1 = HomoLR(name="homo_lr_1")
local_baseline_0 = LocalBaseline(name="local_baseline_0", model_name="LogisticRegression",
model_opts={"penalty": "l2", "tol": 0.0001, "C": 1.0, "fit_intercept": True,
"solver": "lbfgs", "max_iter": 5, "multi_class": "ovr"})
local_baseline_0.get_party_instance(role='guest', party_id=guest).component_param(need_run=True)
local_baseline_0.get_party_instance(role='host', party_id=host).component_param(need_run=True)
local_baseline_1 = LocalBaseline(name="local_baseline_1")
homo_secureboost_0 = HomoSecureBoost(name="homo_secureboost_0", num_trees=3)
homo_secureboost_1 = HomoSecureBoost(name="homo_secureboost_1", num_trees=3)
evaluation_0 = Evaluation(name="evaluation_0")
evaluation_1 = Evaluation(name="evaluation_1")
pipeline.add_component(reader_0)
pipeline.add_component(reader_1)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data),
model=Model(model=data_transform_0.output.model))
pipeline.add_component(federated_sample_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(homo_binning_0, data=Data(data=federated_sample_0.output.data))
pipeline.add_component(homo_binning_1, data=Data(data=data_transform_1.output.data),
model=Model(model=homo_binning_0.output.model))
pipeline.add_component(homo_onehot_0, data=Data(data=homo_binning_0.output.data))
pipeline.add_component(homo_onehot_1, data=Data(data=homo_binning_1.output.data),
model=Model(model=homo_onehot_0.output.model))
pipeline.add_component(homo_lr_0, data=Data(data=homo_onehot_0.output.data))
pipeline.add_component(homo_lr_1, data=Data(data=homo_onehot_1.output.data),
model=Model(model=homo_lr_0.output.model))
pipeline.add_component(local_baseline_0, data=Data(data=homo_onehot_0.output.data))
pipeline.add_component(local_baseline_1, data=Data(data=homo_onehot_1.output.data),
model=Model(model=local_baseline_0.output.model))
pipeline.add_component(homo_secureboost_0, data=Data(data=homo_onehot_0.output.data))
pipeline.add_component(homo_secureboost_1, data=Data(data=homo_onehot_1.output.data),
model=Model(model=homo_secureboost_0.output.model))
pipeline.add_component(evaluation_0,
data=Data(
data=[homo_lr_0.output.data, homo_lr_1.output.data,
local_baseline_0.output.data, local_baseline_1.output.data]))
pipeline.add_component(evaluation_1,
data=Data(
data=[homo_secureboost_0.output.data, homo_secureboost_1.output.data]))
pipeline.compile()
pipeline.fit()
print(pipeline.get_component("evaluation_0").get_summary())
print(pipeline.get_component("evaluation_1").get_summary())
