def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {
"name": "dvisits_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "dvisits_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0", with_label=False)
intersection_0 = Intersection(name="intersection_0")
feature_imputation_0 = FeatureImputation(name="feature_imputation_0",
default_value=42,
missing_impute=[0])
feature_imputation_0.get_party_instance(
role='guest', party_id=guest).component_param(col_missing_fill_method={
"doctorco": "min",
"hscore": "designated"
})
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(feature_imputation_0,
data=Data(data=intersection_0.output.data))
pipeline.compile()
pipeline.fit()
# predict
# deploy required components
pipeline.deploy_component(
[data_transform_0, intersection_0, feature_imputation_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_0)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(
pipeline,
data=Data(predict_input={
pipeline.data_transform_0.input.data: reader_0.output.data
}))
# run predict model
predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {
"name": "nus_wide_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "nus_wide_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
hetero_ftl_0 = HeteroFTL(name='hetero_ftl_0',
epochs=10,
alpha=1,
batch_size=-1,
mode='plain')
hetero_ftl_0.add_nn_layer(
Dense(units=32,
activation='sigmoid',
kernel_initializer=initializers.RandomNormal(stddev=1.0),
bias_initializer=initializers.Zeros()))
hetero_ftl_0.compile(optimizer=optimizers.Adam(lr=0.01))
evaluation_0 = Evaluation(name='evaluation_0', eval_type="binary")
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(hetero_ftl_0,
data=Data(train_data=data_transform_0.output.data))
pipeline.add_component(evaluation_0,
data=Data(data=hetero_ftl_0.output.data))
pipeline.compile()
pipeline.fit()
# predict
# deploy required components
pipeline.deploy_component([data_transform_0, hetero_ftl_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_0)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(
pipeline,
data=Data(predict_input={
pipeline.data_transform_0.input.data: reader_0.output.data
}))
# run predict model
predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
label_name="y",
label_type="int",
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
sample_weight_0 = SampleWeight(name="sample_weight_0")
sample_weight_0.get_party_instance(
role='guest', party_id=guest).component_param(need_run=True,
class_weight="balanced")
sample_weight_0.get_party_instance(
role='host', party_id=host).component_param(need_run=False)
sample_weight_1 = SampleWeight(name="sample_weight_1")
hetero_lr_0 = HeteroLR(name="hetero_lr_0",
optimizer="nesterov_momentum_sgd",
tol=0.001,
alpha=0.01,
max_iter=20,
early_stop="weight_diff",
batch_size=-1,
learning_rate=0.15,
init_param={"init_method": "zeros"})
evaluation_0 = Evaluation(name="evaluation_0",
eval_type="binary",
pos_label=1)
# evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(sample_weight_0,
data=Data(data=intersection_0.output.data))
pipeline.add_component(sample_weight_1,
data=Data(data=intersection_0.output.data),
model=Model(model=sample_weight_0.output.model))
pipeline.add_component(hetero_lr_0,
data=Data(train_data=sample_weight_1.output.data))
pipeline.add_component(evaluation_0,
data=Data(data=hetero_lr_0.output.data))
pipeline.compile()
pipeline.fit()
# predict
# deploy required components
pipeline.deploy_component(
[data_transform_0, intersection_0, sample_weight_0, hetero_lr_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_0)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(
pipeline,
data=Data(predict_input={
pipeline.data_transform_0.input.data: reader_0.output.data
}))
# run predict model
predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""):
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
hosts = parties.host[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role='guest', party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=hosts)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(
role='host', party_id=hosts).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0",
output_format='dense')
# get DataTransform party instance of guest
data_transform_0_guest_party_instance = data_transform_0.get_party_instance(
role='guest', party_id=guest)
# configure DataTransform for guest
data_transform_0_guest_party_instance.component_param(with_label=True)
# get and configure DataTransform party instance of host
data_transform_0.get_party_instance(
role='host', party_id=hosts).component_param(with_label=False)
# define Intersection components
intersection_0 = Intersection(name="intersection_0")
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
lr_param = {
"name": "hetero_sshe_lr_0",
"penalty": "L2",
"optimizer": "sgd",
"tol": 0.0001,
"alpha": 0.01,
"max_iter": 30,
"early_stop": "weight_diff",
"batch_size": -1,
"learning_rate": 0.15,
"init_param": {
"init_method": "zeros",
"fit_intercept": False
},
"encrypt_param": {
"key_length": 1024
},
"reveal_every_iter": False,
"reveal_strategy": "respectively"
}
hetero_sshe_lr_0 = HeteroSSHELR(**lr_param)
pipeline.add_component(hetero_sshe_lr_0,
data=Data(train_data=intersection_0.output.data))
evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary")
pipeline.add_component(evaluation_0,
data=Data(data=hetero_sshe_lr_0.output.data))
pipeline.compile()
# fit model
pipeline.fit()
# query component summary
prettify(pipeline.get_component("hetero_sshe_lr_0").get_summary())
prettify(pipeline.get_component("evaluation_0").get_summary())
pipeline.deploy_component(
[data_transform_0, intersection_0, hetero_sshe_lr_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_0)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(
pipeline,
data=Data(predict_input={
pipeline.data_transform_0.input.data: reader_0.output.data
}))
# run predict model
predict_pipeline.predict()
return pipeline
def main(config="../../config.yaml", namespace=""):
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
hosts = parties.host[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
# guest_train_data = {"name": "default_credit_hetero_guest", "namespace": f"experiment{namespace}"}
# host_train_data = {"name": "default_credit_hetero_host", "namespace": f"experiment{namespace}"}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role='guest', party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=hosts)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(
role='host', party_id=hosts).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0",
output_format='dense')
# get DataTransform party instance of guest
data_transform_0_guest_party_instance = data_transform_0.get_party_instance(
role='guest', party_id=guest)
# configure DataTransform for guest
data_transform_0_guest_party_instance.component_param(with_label=True)
# get and configure DataTransform party instance of host
data_transform_0.get_party_instance(
role='host', party_id=hosts).component_param(with_label=False)
# define Intersection components
intersection_0 = Intersection(name="intersection_0")
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
statistic_param = {
"name": "statistic_0",
"statistics": ["95%", "coefficient_of_variance", "stddev"],
"column_indexes": [1, 2],
"column_names": []
}
statistic_0 = DataStatistics(**statistic_param)
pipeline.add_component(statistic_0,
data=Data(data=intersection_0.output.data))
pipeline.compile()
# fit model
pipeline.fit()
# query component summary
prettify(pipeline.get_component("statistic_0").get_summary())
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True)
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_pearson_0 = HeteroPearson(name='hetero_pearson_0',
column_indexes=-1)
hetero_binning_0 = HeteroFeatureBinning(name='hetero_binning_0')
selection_param = {
"name": "hetero_feature_selection_0",
"select_col_indexes": -1,
"select_names": [],
"filter_methods": ["vif_filter", "correlation_filter"],
"vif_param": {
"threshold": 5
},
"correlation_param": {
"threshold": 0.5,
"select_federated": False
}
}
hetero_feature_selection_0 = HeteroFeatureSelection(**selection_param)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_binning_0,
data=Data(data=intersection_0.output.data))
pipeline.add_component(hetero_pearson_0,
data=Data(data=intersection_0.output.data))
pipeline.add_component(
hetero_feature_selection_0,
data=Data(data=intersection_0.output.data),
model=Model(isometric_model=[
hetero_pearson_0.output.model, hetero_binning_0.output.model
]))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "breast_homo_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_homo_host",
"namespace": f"experiment{namespace}"
}
guest_eval_data = {
"name": "breast_homo_guest",
"namespace": f"experiment{namespace}"
}
host_eval_data = {
"name": "breast_homo_host",
"namespace": f"experiment{namespace}"
}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role='guest', party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=host, arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
reader_1 = Reader(name="reader_1")
reader_1.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_eval_data)
reader_1.get_party_instance(
role='host', party_id=host).component_param(table=host_eval_data)
# define DataTransform components
data_transform_0 = DataTransform(
name="data_transform_0", with_label=True,
output_format="dense") # start component numbering at 0
data_transform_1 = DataTransform(
name="data_transform_1") # start component numbering at 0
scale_0 = FeatureScale(name='scale_0')
scale_1 = FeatureScale(name='scale_1')
param = {
"penalty": "L2",
"optimizer": "sgd",
"tol": 1e-05,
"alpha": 0.01,
"max_iter": 3,
"early_stop": "diff",
"batch_size": 320,
"learning_rate": 0.15,
"callback_param": {
"callbacks": ["EarlyStopping"],
"validation_freqs": 1
},
"init_param": {
"init_method": "zeros"
},
"encrypt_param": {
"method": None
},
"cv_param": {
"n_splits": 4,
"shuffle": True,
"random_seed": 33,
"need_cv": False
}
}
homo_lr_0 = HomoLR(name='homo_lr_0', **param)
# add components to pipeline, in order of task execution
pipeline.add_component(reader_0)
pipeline.add_component(reader_1)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(data_transform_1,
data=Data(data=reader_1.output.data),
model=Model(data_transform_0.output.model))
# set data input sources of intersection components
pipeline.add_component(scale_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(scale_1,
data=Data(data=data_transform_1.output.data),
model=Model(scale_0.output.model))
pipeline.add_component(homo_lr_0,
data=Data(train_data=scale_0.output.data,
validate_data=scale_1.output.data))
evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary")
evaluation_0.get_party_instance(
role='host', party_id=host).component_param(need_run=False)
pipeline.add_component(evaluation_0, data=Data(data=homo_lr_0.output.data))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
pipeline.fit()
# query component summary
print(
json.dumps(pipeline.get_component("homo_lr_0").get_summary(),
indent=4,
ensure_ascii=False))
print(
json.dumps(pipeline.get_component("evaluation_0").get_summary(),
indent=4,
ensure_ascii=False))
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
reader_1 = Reader(name="reader_1")
reader_1.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_1.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_1 = DataTransform(name="data_transform_1")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
data_transform_1.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
output_format="dense")
data_transform_1.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
intersect_0 = Intersection(name="intersection_0")
intersect_1 = Intersection(name="intersection_1")
union_0 = Union(name="union_0")
hetero_lr_0 = HeteroLR(name="hetero_lr_0",
max_iter=3,
early_stop="weight_diff",
optimizer="nesterov_momentum_sgd",
tol=1E-4,
alpha=0.01,
learning_rate=0.15,
init_param={"init_method": "random_uniform"})
evaluation_0 = Evaluation(name="evaluation_0",
eval_type="binary",
pos_label=1)
evaluation_0.get_party_instance(
role='host', party_id=host).component_param(need_run=False)
pipeline.add_component(reader_0)
pipeline.add_component(reader_1)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(data_transform_1,
data=Data(data=reader_1.output.data),
model=Model(data_transform_0.output.model))
pipeline.add_component(intersect_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(intersect_1,
data=Data(data=data_transform_1.output.data))
pipeline.add_component(
union_0,
data=Data(data=[intersect_0.output.data, intersect_1.output.data]))
pipeline.add_component(hetero_lr_0,
data=Data(train_data=union_0.output.data))
pipeline.add_component(evaluation_0,
data=Data(data=hetero_lr_0.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "motor_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "motor_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0",
output_format="sparse",
missing_fill=True,
outlier_replace=False)
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
label_name="motor_speed",
label_type="float")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_linr_0 = HeteroLinR(name="hetero_linr_0",
penalty="L2",
optimizer="sqn",
tol=0.001,
alpha=0.01,
max_iter=2,
early_stop="weight_diff",
batch_size=100,
learning_rate=0.15,
decay=0.0,
decay_sqrt=False,
init_param={"init_method": "zeros"},
sqn_param={
"update_interval_L": 3,
"memory_M": 5,
"sample_size": 5000,
"random_seed": None
})
evaluation_0 = Evaluation(name="evaluation_0",
eval_type="regression",
pos_label=1)
# evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_linr_0,
data=Data(train_data=intersection_0.output.data))
pipeline.add_component(evaluation_0,
data=Data(data=hetero_linr_0.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
hosts = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role='guest', party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=hosts, arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(
role='host', party_id=hosts).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0",
output_format='dense')
# get DataTransform party instance of guest
data_transform_0_guest_party_instance = data_transform_0.get_party_instance(
role='guest', party_id=guest)
# configure DataTransform for guest
data_transform_0_guest_party_instance.component_param(with_label=True)
# get and configure DataTransform party instance of host
data_transform_0.get_party_instance(
role='host', party_id=hosts).component_param(with_label=False)
# define Intersection components
intersection_0 = Intersection(name="intersection_0")
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
# secure boost component
hetero_secure_boost_0 = HeteroSecureBoost(
name="hetero_secure_boost_0",
num_trees=3,
task_type="classification",
objective_param={"objective": "cross_entropy"},
encrypt_param={"method": "Paillier"},
tree_param={"max_depth": 3},
validation_freqs=1)
hetero_secure_boost_1 = HeteroSecureBoost(
name="hetero_secure_boost_1",
num_trees=3,
task_type="classification",
objective_param={"objective": "cross_entropy"},
encrypt_param={"method": "Paillier"},
tree_param={"max_depth": 3},
validation_freqs=1)
pipeline.add_component(hetero_secure_boost_0,
data=Data(train_data=intersection_0.output.data))
pipeline.add_component(
hetero_secure_boost_1,
data=Data(train_data=intersection_0.output.data),
model=Model(model=hetero_secure_boost_0.output.model))
evaluation_0 = Evaluation(name="evaluation_0", eval_type="binary")
pipeline.add_component(evaluation_0,
data=Data(data=hetero_secure_boost_1.output.data))
pipeline.compile()
# fit model
pipeline.fit()
# query component summary
prettify(pipeline.get_component("hetero_secure_boost_0").get_summary())
prettify(pipeline.get_component("hetero_secure_boost_1").get_summary())
prettify(pipeline.get_component("evaluation_0").get_summary())
return pipeline
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment_sid{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment_sid{namespace}"
}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role="guest",
party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(
role="guest", party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role="host", party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(
name="data_transform_0",
with_match_id=True) # start component numbering at 0
data_transform_0_guest_party_instance = data_transform_0.get_party_instance(
role="guest", party_id=guest)
# configure DataTransform for guest
data_transform_0_guest_party_instance.component_param(
with_label=True, output_format="dense")
data_transform_0.get_party_instance(
role="host", party_id=host).component_param(with_label=False,
output_format="dense")
intersection_0 = Intersection(name="intersection_0",
join_method="left_join",
sample_id_generator="host")
label_transform_0 = LabelTransform(name="label_transform_0")
label_transform_0.get_party_instance(
role="host", party_id=host).component_param(need_run=False)
hetero_lr_0 = HeteroLR(name="hetero_lr_0",
penalty="L2",
optimizer="sgd",
tol=0.001,
alpha=0.01,
max_iter=20,
early_stop="weight_diff",
batch_size=-1,
learning_rate=0.15,
decay=0.0,
decay_sqrt=False,
init_param={"init_method": "zeros"},
encrypted_mode_calculator_param={"mode": "fast"},
floating_point_precision=23)
label_transform_1 = LabelTransform(name="label_transform_1")
evaluation_0 = Evaluation(name="evaluation_0",
eval_type="binary",
pos_label=1)
# add components to pipeline, in order of task execution
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(label_transform_0,
data=Data(data=intersection_0.output.data))
pipeline.add_component(hetero_lr_0,
data=Data(train_data=label_transform_0.output.data))
pipeline.add_component(label_transform_1,
data=Data(data=hetero_lr_0.output.data),
model=Model(label_transform_0.output.model))
pipeline.add_component(evaluation_0,
data=Data(data=label_transform_1.output.data))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "motor_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "motor_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
label_name="motor_speed",
label_type="float",
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_data_split_0 = HeteroDataSplit(name="hetero_data_split_0",
stratified=True,
test_size=0.3,
validate_size=0.2,
split_points=[0.0, 0.2])
hetero_linr_0 = HeteroLinR(name="hetero_linr_0",
penalty="L2",
optimizer="sgd",
tol=0.001,
alpha=0.01,
max_iter=10,
early_stop="weight_diff",
batch_size=-1,
learning_rate=0.15,
decay=0.0,
decay_sqrt=False,
init_param={"init_method": "zeros"})
hetero_linr_1 = HeteroLinR()
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_data_split_0,
data=Data(data=intersection_0.output.data))
pipeline.add_component(
hetero_linr_0,
data=Data(train_data=hetero_data_split_0.output.data.train_data,
validate_data=hetero_data_split_0.output.data.validate_data))
pipeline.add_component(
hetero_linr_1,
data=Data(test_data=hetero_data_split_0.output.data.test_data),
model=Model(model=hetero_linr_0.output.model))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "dvisits_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "dvisits_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
label_name="doctorco",
label_type="float",
output_format="dense",
missing_fill=True,
outlier_replace=False)
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False,
output_format="dense",
outlier_replace=False)
intersection_0 = Intersection(name="intersection_0")
hetero_poisson_0 = HeteroPoisson(name="hetero_poisson_0",
early_stop="weight_diff",
max_iter=10,
optimizer="rmsprop",
tol=0.001,
penalty="L2",
alpha=100.0,
batch_size=-1,
learning_rate=0.01,
exposure_colname="exposure",
decay_sqrt=False,
init_param={"init_method": "zeros"},
cv_param={
"n_splits": 5,
"shuffle": False,
"random_seed": 103,
"need_cv": True
})
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_poisson_0,
data=Data(train_data=intersection_0.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "breast_homo_guest",
"namespace": f"experiment{namespace}"
}
guest_validate_data = {
"name": "breast_homo_test",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_homo_host",
"namespace": f"experiment{namespace}"
}
host_validate_data = {
"name": "breast_homo_test",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
data_transform_0, data_transform_1 = DataTransform(
name="data_transform_0"), DataTransform(name='data_transform_1')
reader_0, reader_1 = Reader(name="reader_0"), Reader(name='reader_1')
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=True,
output_format="dense")
reader_1.get_party_instance(
role='guest',
party_id=guest).component_param(table=guest_validate_data)
reader_1.get_party_instance(
role='host', party_id=host).component_param(table=host_validate_data)
data_transform_1.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
output_format="dense")
data_transform_1.get_party_instance(
role='host', party_id=host).component_param(with_label=True,
output_format="dense")
homo_secureboost_0 = HomoSecureBoost(
name="homo_secureboost_0",
num_trees=3,
task_type='classification',
objective_param={"objective": "cross_entropy"},
tree_param={"max_depth": 3},
validation_freqs=1)
evaluation_0 = Evaluation(name='evaluation_0', eval_type='binary')
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(reader_1)
pipeline.add_component(data_transform_1,
data=Data(data=reader_1.output.data),
model=Model(data_transform_0.output.model))
pipeline.add_component(homo_secureboost_0,
data=Data(
train_data=data_transform_0.output.data,
validate_data=data_transform_1.output.data))
pipeline.add_component(evaluation_0,
data=Data(homo_secureboost_0.output.data))
pipeline.compile()
pipeline.fit()
# predict
# deploy required components
pipeline.deploy_component([data_transform_0, homo_secureboost_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_1)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(
pipeline,
data=Data(predict_input={
pipeline.data_transform_0.input.data: reader_1.output.data
}))
# run predict model
predict_pipeline.predict()
def main(config="../../config.yaml", param="./vehicle_config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
if isinstance(param, str):
param = JobConfig.load_from_file(param)
assert isinstance(param, dict)
data_set = param.get("data_guest").split('/')[-1]
if data_set == "vehicle_scale_hetero_guest.csv":
guest_data_table = 'vehicle_scale_hetero_guest'
host_data_table = 'vehicle_scale_hetero_host'
else:
raise ValueError(f"Cannot recognized data_set: {data_set}")
guest_train_data = {"name": guest_data_table, "namespace": f"experiment{namespace}"}
host_train_data = {"name": host_data_table, "namespace": f"experiment{namespace}"}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role='guest', party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=host, arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
# define DataTransform components
data_transform_0 = DataTransform(name="data_transform_0") # start component numbering at 0
# get DataTransform party instance of guest
data_transform_0_guest_party_instance = data_transform_0.get_party_instance(role='guest', party_id=guest)
# configure DataTransform for guest
data_transform_0_guest_party_instance.component_param(with_label=True, output_format="dense")
# get and configure DataTransform party instance of host
data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False)
# define Intersection component
intersection_0 = Intersection(name="intersection_0")
lr_param = {
}
config_param = {
"penalty": param["penalty"],
"max_iter": param["max_iter"],
"alpha": param["alpha"],
"learning_rate": param["learning_rate"],
"optimizer": param["optimizer"],
"batch_size": param["batch_size"],
"early_stop": "diff",
"init_param": {
"init_method": param.get("init_method", 'random_uniform'),
"random_seed": param.get("random_seed", 103)
}
}
lr_param.update(config_param)
print(f"lr_param: {lr_param}, data_set: {data_set}")
hetero_lr_0 = HeteroLR(name='hetero_lr_0', **lr_param)
hetero_lr_1 = HeteroLR(name='hetero_lr_1')
evaluation_0 = Evaluation(name='evaluation_0', eval_type="multi")
# add components to pipeline, in order of task execution
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_lr_0, data=Data(train_data=intersection_0.output.data))
pipeline.add_component(hetero_lr_1, data=Data(test_data=intersection_0.output.data),
model=Model(hetero_lr_0.output.model))
pipeline.add_component(evaluation_0, data=Data(data=hetero_lr_0.output.data))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
pipeline.fit()
# query component summary
result_summary = parse_summary_result(pipeline.get_component("evaluation_0").get_summary())
lr_0_data = pipeline.get_component("hetero_lr_0").get_output_data().get("data")
lr_1_data = pipeline.get_component("hetero_lr_1").get_output_data().get("data")
lr_0_score_label = extract_data(lr_0_data, "predict_result", keep_id=True)
lr_1_score_label = extract_data(lr_1_data, "predict_result", keep_id=True)
metric_lr = {
"score_diversity_ratio": classification_metric.Distribution.compute(lr_0_score_label, lr_1_score_label)}
result_summary["distribution_metrics"] = {"hetero_lr": metric_lr}
data_summary = {"train": {"guest": guest_train_data["name"], "host": host_train_data["name"]},
"test": {"guest": guest_train_data["name"], "host": host_train_data["name"]}
}
return data_summary, result_summary
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {"name": "default_credit_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "default_credit_hetero_host", "namespace": f"experiment{namespace}"}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role="guest", party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=host, arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data)
# define DataTransform components
data_transform_0 = DataTransform(name="data_transform_0") # start component numbering at 0
# get DataTransform party instance of guest
data_transform_0_guest_party_instance = data_transform_0.get_party_instance(role="guest", party_id=guest)
# configure DataTransform for guest
data_transform_0_guest_party_instance.component_param(with_label=True, output_format="dense")
# get and configure DataTransform party instance of host
data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False)
# define Intersection components
intersection_0 = Intersection(name="intersection_0", intersect_method="rsa",
sync_intersect_ids=True, only_output_key=False)
param = {
"penalty": "L2",
"optimizer": "nesterov_momentum_sgd",
"tol": 0.0001,
"alpha": 0.01,
"max_iter": 5,
"early_stop": "weight_diff",
"batch_size": -1,
"learning_rate": 0.15,
"init_param": {
"init_method": "random_uniform"
},
"sqn_param": {
"update_interval_L": 3,
"memory_M": 5,
"sample_size": 5000,
"random_seed": None
}
}
hetero_lr_0 = HeteroLR(name="hetero_lr_0", **param)
# define Scorecard component
scorecard_0 = Scorecard(name="scorecard_0")
scorecard_0.get_party_instance(role="guest", party_id=guest).component_param(need_run=True,
method="credit",
offset=500,
factor=20,
factor_base=2,
upper_limit_ratio=3,
lower_limit_value=0)
scorecard_0.get_party_instance(role="host", party_id=host).component_param(need_run=False)
# add components to pipeline, in order of task execution
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
# set data input sources of intersection components
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_lr_0, data=Data(train_data=intersection_0.output.data))
pipeline.add_component(scorecard_0, data=Data(data=hetero_lr_0.output.data))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=False,
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False,
output_format="dense")
param_0 = {
"intersect_method": "rsa",
"rsa_params": {
"hash_method": "sha256",
"final_hash_method": "sha256",
"key_length": 2048
},
"run_cache": True
}
param_1 = {
"intersect_method": "rsa",
"sync_intersect_ids": False,
"only_output_key": True,
"rsa_params": {
"hash_method": "sha256",
"final_hash_method": "sha256",
"key_length": 2048
}
}
intersect_0 = Intersection(name="intersect_0", **param_0)
intersect_1 = Intersection(name="intersect_1", **param_1)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersect_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(intersect_1,
data=Data(data=data_transform_0.output.data),
cache=Cache(intersect_0.output.cache))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
hosts = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "vehicle_scale_homo_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "vehicle_scale_homo_host",
"namespace": f"experiment{namespace}"
}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role='guest', party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=hosts, arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(
role='host', party_id=hosts).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0",
output_format='dense',
with_label=True)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
lr_param = {
"penalty": "L2",
"optimizer": "sgd",
"tol": 1e-05,
"alpha": 0.01,
"early_stop": "diff",
"batch_size": -1,
"learning_rate": 0.15,
"decay": 1,
"decay_sqrt": True,
"init_param": {
"init_method": "zeros"
},
"encrypt_param": {
"method": None
},
"cv_param": {
"n_splits": 4,
"shuffle": True,
"random_seed": 33,
"need_cv": False
},
"callback_param": {
"callbacks": ["ModelCheckpoint", "EarlyStopping"]
}
}
homo_lr_0 = HomoLR(name="homo_lr_0", max_iter=1, **lr_param)
homo_lr_1 = HomoLR(name="homo_lr_1")
pipeline.add_component(homo_lr_0,
data=Data(train_data=data_transform_0.output.data))
pipeline.add_component(homo_lr_1,
data=Data(test_data=data_transform_0.output.data),
model=Model(model=homo_lr_0.output.model))
evaluation_0 = Evaluation(name="evaluation_0", eval_type="multi")
pipeline.add_component(
evaluation_0,
data=Data(data=[homo_lr_0.output.data, homo_lr_1.output.data]))
pipeline.compile()
# fit model
pipeline.fit()
# query component summary
prettify(pipeline.get_component("evaluation_0").get_summary())
return pipeline
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {
"name": "motor_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "motor_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
label_name="motor_speed",
label_type="float",
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
sample_weight_0 = SampleWeight(name="sample_weight_0")
sample_weight_0.get_party_instance(
role='guest', party_id=guest).component_param(need_run=True,
sample_weight_name="pm")
sample_weight_0.get_party_instance(
role='host', party_id=host).component_param(need_run=False)
hetero_linr_0 = HeteroSSHELinR(name="hetero_linr_0",
penalty="L2",
optimizer="rmsprop",
tol=0.001,
alpha=0.01,
max_iter=20,
early_stop="weight_diff",
batch_size=-1,
learning_rate=0.15,
decay=0.0,
decay_sqrt=False,
init_param={"init_method": "zeros"},
reveal_every_iter=True,
reveal_strategy="respectively")
evaluation_0 = Evaluation(name="evaluation_0",
eval_type="regression",
pos_label=1)
# evaluation_0.get_party_instance(role='host', party_id=host).component_param(need_run=False)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(sample_weight_0,
data=Data(data=intersection_0.output.data))
pipeline.add_component(hetero_linr_0,
data=Data(train_data=sample_weight_0.output.data))
pipeline.add_component(evaluation_0,
data=Data(data=hetero_linr_0.output.data))
pipeline.compile()
pipeline.fit()
# predict
# deploy required components
pipeline.deploy_component(
[data_transform_0, intersection_0, hetero_linr_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_0)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(
pipeline,
data=Data(predict_input={
pipeline.data_transform_0.input.data: reader_0.output.data
}))
# run predict model
predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role='guest', party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=host, arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
# define DataTransform components
data_transform_0 = DataTransform(
name="data_transform_0", with_label=True,
output_format="dense") # start component numbering at 0
data_transform_0.get_party_instance(
role="host", party_id=host).component_param(with_label=False)
intersect_0 = Intersection(name='intersect_0')
scale_0 = FeatureScale(name='scale_0', need_run=False)
sample_weight_0 = SampleWeight(name="sample_weight_0",
class_weight={
"0": 1,
"1": 2
})
sample_weight_0.get_party_instance(
role="host", party_id=host).component_param(need_run=False)
param = {
"penalty": None,
"optimizer": "sgd",
"tol": 1e-05,
"alpha": 0.01,
"max_iter": 3,
"early_stop": "weight_diff",
"batch_size": 320,
"learning_rate": 0.15,
"decay": 0,
"decay_sqrt": True,
"init_param": {
"init_method": "ones"
},
"reveal_every_iter": False,
"reveal_strategy": "respectively"
}
hetero_sshe_lr_0 = HeteroSSHELR(name='hetero_sshe_lr_0', **param)
evaluation_0 = Evaluation(name='evaluation_0')
# add components to pipeline, in order of task execution
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersect_0,
data=Data(data=data_transform_0.output.data))
# set data input sources of intersection components
pipeline.add_component(scale_0, data=Data(data=intersect_0.output.data))
pipeline.add_component(sample_weight_0,
data=Data(data=scale_0.output.data))
pipeline.add_component(hetero_sshe_lr_0,
data=Data(train_data=sample_weight_0.output.data))
pipeline.add_component(evaluation_0,
data=Data(data=hetero_sshe_lr_0.output.data))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
pipeline.fit()
# query component summary
print(
json.dumps(pipeline.get_component("evaluation_0").get_summary(),
indent=4,
ensure_ascii=False))
def main(config="../../config.yaml", param="./linr_config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
if isinstance(param, str):
param = JobConfig.load_from_file(param)
guest_train_data = {"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "motor_hetero_host", "namespace": f"experiment{namespace}"}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role='guest', party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=host, arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
# define DataTransform components
data_transform_0 = DataTransform(name="data_transform_0") # start component numbering at 0
# get DataTransform party instance of guest
data_transform_0_guest_party_instance = data_transform_0.get_party_instance(role='guest', party_id=guest)
# configure DataTransform for guest
data_transform_0_guest_party_instance.component_param(with_label=True, output_format="dense",
label_name=param["label_name"], label_type="float")
# get and configure DataTransform party instance of host
data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False)
# define Intersection component
intersection_0 = Intersection(name="intersection_0")
param = {
"penalty": param["penalty"],
"max_iter": param["max_iter"],
"optimizer": param["optimizer"],
"learning_rate": param["learning_rate"],
"init_param": param["init_param"],
"batch_size": param["batch_size"],
"alpha": param["alpha"]
}
hetero_linr_0 = HeteroLinR(name='hetero_linr_0', **param)
hetero_linr_1 = HeteroLinR(name='hetero_linr_1')
evaluation_0 = Evaluation(name='evaluation_0', eval_type="regression",
metrics=["r2_score",
"mean_squared_error",
"root_mean_squared_error",
"explained_variance"])
# add components to pipeline, in order of task execution
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data))
pipeline.add_component(hetero_linr_1,data=Data(test_data=intersection_0.output.data),
model=Model(hetero_linr_0.output.model))
pipeline.add_component(evaluation_0, data=Data(data=hetero_linr_0.output.data))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
pipeline.fit()
metric_summary = parse_summary_result(pipeline.get_component("evaluation_0").get_summary())
data_linr_0 = extract_data(pipeline.get_component("hetero_linr_0").get_output_data().get("data"), "predict_result")
data_linr_1 = extract_data(pipeline.get_component("hetero_linr_1").get_output_data().get("data"), "predict_result")
desc_linr_0 = regression_metric.Describe().compute(data_linr_0)
desc_linr_1 = regression_metric.Describe().compute(data_linr_1)
metric_summary["script_metrics"] = {"linr_train": desc_linr_0,
"linr_validate": desc_linr_1}
data_summary = {"train": {"guest": guest_train_data["name"], "host": host_train_data["name"]},
"test": {"guest": guest_train_data["name"], "host": host_train_data["name"]}
}
return data_summary, metric_summary
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
reader_1 = Reader(name="reader_1")
reader_1.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_1.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
reader_2 = Reader(name="reader_2")
reader_2.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_2.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
missing_fill=True,
outlier_replace=True)
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False,
missing_fill=True,
outlier_replace=True)
data_transform_1 = DataTransform(name="data_transform_1")
data_transform_2 = DataTransform(name="data_transform_2")
intersection_0 = Intersection(name="intersection_0")
intersection_1 = Intersection(name="intersection_1")
intersection_2 = Intersection(name="intersection_2")
union_0 = Union(name="union_0")
federated_sample_0 = FederatedSample(name="federated_sample_0",
mode="stratified",
method="downsample",
fractions=[[0, 1.0], [1, 1.0]])
feature_scale_0 = FeatureScale(name="feature_scale_0")
feature_scale_1 = FeatureScale(name="feature_scale_1")
hetero_feature_binning_0 = HeteroFeatureBinning(
name="hetero_feature_binning_0")
hetero_feature_binning_1 = HeteroFeatureBinning(
name="hetero_feature_binning_1")
hetero_feature_selection_0 = HeteroFeatureSelection(
name="hetero_feature_selection_0")
hetero_feature_selection_1 = HeteroFeatureSelection(
name="hetero_feature_selection_1")
one_hot_0 = OneHotEncoder(name="one_hot_0")
one_hot_1 = OneHotEncoder(name="one_hot_1")
hetero_lr_0 = HeteroLR(name="hetero_lr_0",
penalty="L2",
optimizer="rmsprop",
tol=1e-5,
init_param={"init_method": "random_uniform"},
alpha=0.01,
max_iter=3,
early_stop="diff",
batch_size=320,
learning_rate=0.15)
hetero_lr_1 = HeteroLR(name="hetero_lr_1")
hetero_lr_2 = HeteroLR(name="hetero_lr_2",
penalty="L2",
optimizer="rmsprop",
tol=1e-5,
init_param={"init_method": "random_uniform"},
alpha=0.01,
max_iter=3,
early_stop="diff",
batch_size=320,
learning_rate=0.15,
cv_param={
"n_splits": 5,
"shuffle": True,
"random_seed": 103,
"need_cv": True
})
hetero_sshe_lr_0 = HeteroSSHELR(
name="hetero_sshe_lr_0",
reveal_every_iter=True,
reveal_strategy="respectively",
penalty="L2",
optimizer="rmsprop",
tol=1e-5,
batch_size=320,
learning_rate=0.15,
init_param={"init_method": "random_uniform"},
alpha=0.01,
max_iter=3)
hetero_sshe_lr_1 = HeteroSSHELR(name="hetero_sshe_lr_1")
local_baseline_0 = LocalBaseline(name="local_baseline_0",
model_name="LogisticRegression",
model_opts={
"penalty": "l2",
"tol": 0.0001,
"C": 1.0,
"fit_intercept": True,
"solver": "lbfgs",
"max_iter": 5,
"multi_class": "ovr"
})
local_baseline_0.get_party_instance(
role='guest', party_id=guest).component_param(need_run=True)
local_baseline_0.get_party_instance(
role='host', party_id=host).component_param(need_run=False)
local_baseline_1 = LocalBaseline(name="local_baseline_1")
hetero_secureboost_0 = HeteroSecureBoost(name="hetero_secureboost_0",
num_trees=3)
hetero_secureboost_1 = HeteroSecureBoost(name="hetero_secureboost_1")
hetero_secureboost_2 = HeteroSecureBoost(name="hetero_secureboost_2",
num_trees=3,
cv_param={
"shuffle": False,
"need_cv": True
})
hetero_linr_0 = HeteroLinR(name="hetero_linr_0",
penalty="L2",
optimizer="sgd",
tol=0.001,
alpha=0.01,
max_iter=3,
early_stop="weight_diff",
batch_size=-1,
learning_rate=0.15,
decay=0.0,
decay_sqrt=False,
init_param={"init_method": "zeros"},
floating_point_precision=23)
hetero_linr_1 = HeteroLinR(name="hetero_linr_1")
hetero_sshe_linr_0 = HeteroSSHELinR(name="hetero_sshe_linr_0",
max_iter=5,
early_stop="weight_diff",
batch_size=-1)
hetero_sshe_linr_1 = HeteroSSHELinR(name="hetero_sshe_linr_1")
hetero_poisson_0 = HeteroPoisson(name="hetero_poisson_0",
early_stop="weight_diff",
max_iter=10,
alpha=100.0,
batch_size=-1,
learning_rate=0.01,
optimizer="rmsprop",
exposure_colname="exposure",
decay_sqrt=False,
tol=0.001,
init_param={"init_method": "zeros"},
penalty="L2")
hetero_poisson_1 = HeteroPoisson(name="hetero_poisson_1")
hetero_sshe_poisson_0 = HeteroSSHEPoisson(name="hetero_sshe_poisson_0",
max_iter=5)
hetero_sshe_poisson_1 = HeteroSSHEPoisson(name="hetero_sshe_poisson_1")
evaluation_0 = Evaluation(name="evaluation_0")
evaluation_1 = Evaluation(name="evaluation_1")
evaluation_2 = Evaluation(name="evaluation_2")
pipeline.add_component(reader_0)
pipeline.add_component(reader_1)
pipeline.add_component(reader_2)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(data_transform_1,
data=Data(data=reader_1.output.data),
model=Model(model=data_transform_0.output.model))
pipeline.add_component(data_transform_2,
data=Data(data=reader_2.output.data),
model=Model(model=data_transform_0.output.model))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(intersection_1,
data=Data(data=data_transform_1.output.data))
pipeline.add_component(intersection_2,
data=Data(data=data_transform_2.output.data))
pipeline.add_component(
union_0,
data=Data(
data=[intersection_0.output.data, intersection_2.output.data]))
pipeline.add_component(federated_sample_0,
data=Data(data=intersection_1.output.data))
pipeline.add_component(feature_scale_0,
data=Data(data=union_0.output.data))
pipeline.add_component(feature_scale_1,
data=Data(data=federated_sample_0.output.data),
model=Model(model=feature_scale_0.output.model))
pipeline.add_component(hetero_feature_binning_0,
data=Data(data=feature_scale_0.output.data))
pipeline.add_component(
hetero_feature_binning_1,
data=Data(data=feature_scale_1.output.data),
model=Model(model=hetero_feature_binning_0.output.model))
pipeline.add_component(
hetero_feature_selection_0,
data=Data(data=hetero_feature_binning_0.output.data))
pipeline.add_component(
hetero_feature_selection_1,
data=Data(data=hetero_feature_binning_1.output.data),
model=Model(model=hetero_feature_selection_0.output.model))
pipeline.add_component(
one_hot_0, data=Data(data=hetero_feature_selection_0.output.data))
pipeline.add_component(
one_hot_1,
data=Data(data=hetero_feature_selection_1.output.data),
model=Model(model=one_hot_0.output.model))
pipeline.add_component(hetero_lr_0,
data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(hetero_lr_1,
data=Data(test_data=one_hot_1.output.data),
model=Model(model=hetero_lr_0.output.model))
pipeline.add_component(hetero_lr_2,
data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(local_baseline_0,
data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(local_baseline_1,
data=Data(test_data=one_hot_1.output.data),
model=Model(model=local_baseline_0.output.model))
pipeline.add_component(hetero_sshe_lr_0,
data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(hetero_sshe_lr_1,
data=Data(test_data=one_hot_1.output.data),
model=Model(model=hetero_sshe_lr_0.output.model))
pipeline.add_component(hetero_secureboost_0,
data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(
hetero_secureboost_1,
data=Data(test_data=one_hot_1.output.data),
model=Model(model=hetero_secureboost_0.output.model))
pipeline.add_component(hetero_secureboost_2,
data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(hetero_linr_0,
data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(hetero_linr_1,
data=Data(test_data=one_hot_1.output.data),
model=Model(model=hetero_linr_0.output.model))
pipeline.add_component(hetero_sshe_linr_0,
data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(hetero_sshe_linr_1,
data=Data(test_data=one_hot_1.output.data),
model=Model(model=hetero_sshe_linr_0.output.model))
pipeline.add_component(hetero_poisson_0,
data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(hetero_poisson_1,
data=Data(test_data=one_hot_1.output.data),
model=Model(model=hetero_poisson_0.output.model))
pipeline.add_component(
evaluation_0,
data=Data(data=[
hetero_lr_0.output.data, hetero_lr_1.output.data,
hetero_sshe_lr_0.output.data, hetero_sshe_lr_1.output.data,
local_baseline_0.output.data, local_baseline_1.output.data
]))
pipeline.add_component(hetero_sshe_poisson_0,
data=Data(train_data=one_hot_0.output.data))
pipeline.add_component(
hetero_sshe_poisson_1,
data=Data(test_data=one_hot_1.output.data),
model=Model(model=hetero_sshe_poisson_0.output.model))
pipeline.add_component(
evaluation_1,
data=Data(data=[
hetero_linr_0.output.data, hetero_linr_1.output.data,
hetero_sshe_linr_0.output.data, hetero_linr_1.output.data
]))
pipeline.add_component(
evaluation_2,
data=Data(data=[
hetero_poisson_0.output.data, hetero_poisson_1.output.data,
hetero_sshe_poisson_0.output.data,
hetero_sshe_poisson_1.output.data
]))
pipeline.compile()
pipeline.fit()
print(pipeline.get_component("evaluation_0").get_summary())
print(pipeline.get_component("evaluation_1").get_summary())
print(pipeline.get_component("evaluation_2").get_summary())
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
# data sets
guest_train_data = {"name": "student_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "student_hetero_host", "namespace": f"experiment{namespace}"}
guest_validate_data = {"name": "student_hetero_guest", "namespace": f"experiment{namespace}"}
host_validate_data = {"name": "student_hetero_host", "namespace": f"experiment{namespace}"}
# init pipeline
pipeline = PipeLine().set_initiator(role="guest", party_id=guest).set_roles(guest=guest, host=host,)
# set data reader and data-io
reader_0, reader_1 = Reader(name="reader_0"), Reader(name="reader_1")
reader_0.get_party_instance(role="guest", party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role="host", party_id=host).component_param(table=host_train_data)
reader_1.get_party_instance(role="guest", party_id=guest).component_param(table=guest_validate_data)
reader_1.get_party_instance(role="host", party_id=host).component_param(table=host_validate_data)
data_transform_0, data_transform_1 = DataTransform(name="data_transform_0"), DataTransform(name="data_transform_1")
data_transform_0.get_party_instance(
role="guest",
party_id=guest).component_param(
with_label=True,
output_format="dense",
label_type="float")
data_transform_0.get_party_instance(role="host", party_id=host).component_param(with_label=False)
data_transform_1.get_party_instance(
role="guest",
party_id=guest).component_param(
with_label=True,
output_format="dense",
label_type="float")
data_transform_1.get_party_instance(role="host", party_id=host).component_param(with_label=False)
# data intersect component
intersect_0 = Intersection(name="intersection_0")
intersect_1 = Intersection(name="intersection_1")
# secure boost component
hetero_secure_boost_0 = HeteroSecureBoost(name="hetero_secure_boost_0",
num_trees=3,
task_type="regression",
objective_param={"objective": "lse"},
encrypt_param={"method": "Paillier"},
tree_param={"max_depth": 3},
validation_freqs=1)
# evaluation component
evaluation_0 = Evaluation(name="evaluation_0", eval_type="regression")
pipeline.add_component(reader_0)
pipeline.add_component(reader_1)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(
data_transform_1, data=Data(
data=reader_1.output.data), model=Model(
data_transform_0.output.model))
pipeline.add_component(intersect_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(intersect_1, data=Data(data=data_transform_1.output.data))
pipeline.add_component(hetero_secure_boost_0, data=Data(train_data=intersect_0.output.data,
validate_data=intersect_1.output.data))
pipeline.add_component(evaluation_0, data=Data(data=hetero_secure_boost_0.output.data))
pipeline.compile()
pipeline.fit()
print("fitting hetero secureboost done, result:")
print(pipeline.get_component("hetero_secure_boost_0").get_summary())
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {"name": "vehicle_scale_hetero_guest", "namespace": f"experiment{namespace}"}
guest_validate_data = {"name": "vehicle_scale_hetero_guest", "namespace": f"experiment{namespace}"}
host_train_data = {"name": "vehicle_scale_hetero_host", "namespace": f"experiment{namespace}"}
host_validate_data = {"name": "vehicle_scale_hetero_host", "namespace": f"experiment{namespace}"}
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host)
data_transform_0, data_transform_1 = DataTransform(name="data_transform_0"), DataTransform(name='data_transform_1')
reader_0, reader_1 = Reader(name="reader_0"), Reader(name='reader_1')
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data)
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(
with_label=True, output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(
with_label=False, output_format="dense")
reader_1.get_party_instance(role='guest', party_id=guest).component_param(table=guest_validate_data)
reader_1.get_party_instance(role='host', party_id=host).component_param(table=host_validate_data)
data_transform_1.get_party_instance(
role='guest', party_id=guest).component_param(
with_label=True, output_format="dense")
data_transform_1.get_party_instance(
role='host', party_id=host).component_param(
with_label=True, output_format="dense")
intersection_0 = Intersection(name="intersection_0")
intersection_1 = Intersection(name="intersection_1")
param = {
"method": "quantile",
"optimal_binning_param": {
"metric_method": "gini",
"min_bin_pct": 0.05,
"max_bin_pct": 0.8,
"init_bucket_method": "quantile",
"init_bin_nums": 100,
"mixture": True
},
"compress_thres": 10000,
"head_size": 10000,
"error": 0.001,
"bin_num": 10,
"bin_indexes": -1,
"bin_names": None,
"category_indexes": [0, 1, 2],
"category_names": None,
"adjustment_factor": 0.5,
"local_only": False,
"transform_param": {
"transform_cols": -1,
"transform_names": None,
"transform_type": "bin_num"
}
}
hetero_feature_binning_0 = HeteroFeatureBinning(name="hetero_feature_binning_0", **param)
hetero_feature_binning_1 = HeteroFeatureBinning(name='hetero_feature_binning_1')
selection_param = {
"name": "hetero_feature_selection_0",
"select_col_indexes": -1,
"select_names": [],
"filter_methods": ["iv_filter"],
"iv_param": {
"metrics": ["iv", "iv", "iv"],
"filter_type": ["threshold", "top_k", "top_percentile"],
"threshold": [2, 10, 0.9],
"mul_class_merge_type": ["max", "min", "average"]
}}
hetero_feature_selection_0 = HeteroFeatureSelection(**selection_param)
hetero_feature_selection_1 = HeteroFeatureSelection(name="hetero_feature_selection_1")
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(reader_1)
pipeline.add_component(
data_transform_1, data=Data(
data=reader_1.output.data), model=Model(
data_transform_0.output.model))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(intersection_1, data=Data(data=data_transform_1.output.data))
pipeline.add_component(hetero_feature_binning_0, data=Data(data=intersection_0.output.data))
pipeline.add_component(hetero_feature_binning_1, data=Data(data=intersection_1.output.data),
model=Model(hetero_feature_binning_0.output.model))
pipeline.add_component(hetero_feature_selection_0, data=Data(data=hetero_feature_binning_0.output.data),
model=Model(isometric_model=hetero_feature_binning_0.output.model))
pipeline.add_component(hetero_feature_selection_1, data=Data(data=hetero_feature_binning_1.output.data),
model=Model(hetero_feature_selection_0.output.model))
pipeline.compile()
pipeline.fit()
# predict
# deploy required components
pipeline.deploy_component([data_transform_0, intersection_0, hetero_feature_selection_0])
predict_pipeline = PipeLine()
# add data reader onto predict pipeline
predict_pipeline.add_component(reader_1)
# add selected components from train pipeline onto predict pipeline
# specify data source
predict_pipeline.add_component(
pipeline, data=Data(
predict_input={
pipeline.data_transform_0.input.data: reader_1.output.data}))
# run predict model
predict_pipeline.predict()
def main(config="../../config.yaml", namespace=""):
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(role='guest',
party_id=guest).set_roles(guest=guest,
host=host)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True)
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
param = {
"model_id": "guest-10000#host-9999#model",
"model_version": "202108301602196678300",
"component_name": "hetero_feature_binning_0",
"step_index": None
}
model_loader_0 = ModelLoader(name="model_loader_0", **param)
selection_param = {
"name": "hetero_feature_selection_0",
"select_col_indexes": -1,
"select_names": [],
"filter_methods": ["iv_filter"],
"iv_param": {
"metrics": ["iv", "iv", "iv"],
"filter_type": ["threshold", "top_k", "top_percentile"],
"take_high": True,
"threshold": [0.03, 15, 0.7],
"host_thresholds": [[0.15], None, None],
"select_federated": True
}
}
hetero_feature_selection_0 = HeteroFeatureSelection(**selection_param)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(model_loader_0)
pipeline.add_component(
hetero_feature_selection_0,
data=Data(data=intersection_0.output.data),
model=Model(isometric_model=model_loader_0.output.model))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "dvisits_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "dvisits_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
label_name="doctorco",
label_type="float",
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_poisson_0 = HeteroPoisson(
name="hetero_poisson_0",
early_stop="weight_diff",
max_iter=3,
alpha=100.0,
batch_size=-1,
learning_rate=0.01,
optimizer="rmsprop",
exposure_colname="exposure",
decay_sqrt=False,
tol=0.001,
callback_param={"callbacks": ["ModelCheckpoint"]},
init_param={"init_method": "zeros"},
penalty="L2",
encrypted_mode_calculator_param={"mode": "fast"})
hetero_poisson_1 = HeteroPoisson(
name="hetero_poisson_1",
early_stop="weight_diff",
max_iter=10,
alpha=100.0,
batch_size=-1,
learning_rate=0.01,
optimizer="rmsprop",
exposure_colname="exposure",
decay_sqrt=False,
tol=0.001,
penalty="L2",
encrypted_mode_calculator_param={"mode": "fast"})
evaluation_0 = Evaluation(name="evaluation_0",
eval_type="regression",
pos_label=1)
evaluation_0.get_party_instance(
role='host', party_id=host).component_param(need_run=False)
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_poisson_0,
data=Data(train_data=intersection_0.output.data))
pipeline.add_component(hetero_poisson_1,
data=Data(train_data=intersection_0.output.data),
model=Model(model=hetero_poisson_0.output.model))
pipeline.add_component(evaluation_0,
data=Data(data=hetero_poisson_1.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
guest_eval_data = {
"name": "breast_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_eval_data = {
"name": "breast_hetero_host",
"namespace": f"experiment{namespace}"
}
# initialize pipeline
pipeline = PipeLine()
# set job initiator
pipeline.set_initiator(role='guest', party_id=guest)
# set participants information
pipeline.set_roles(guest=guest, host=host, arbiter=arbiter)
# define Reader components to read in data
reader_0 = Reader(name="reader_0")
# configure Reader for guest
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
# configure Reader for host
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
reader_1 = Reader(name="reader_1")
reader_1.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_eval_data)
reader_1.get_party_instance(
role='host', party_id=host).component_param(table=host_eval_data)
# define DataTransform components
data_transform_0 = DataTransform(
name="data_transform_0") # start component numbering at 0
data_transform_1 = DataTransform(name="data_transform_1")
# get DataTransform party instance of guest
data_transform_0_guest_party_instance = data_transform_0.get_party_instance(
role='guest', party_id=guest)
# configure DataTransform for guest
data_transform_0_guest_party_instance.component_param(
with_label=True, output_format="dense")
# get and configure DataTransform party instance of host
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
# define Intersection components
intersection_0 = Intersection(name="intersection_0")
intersection_1 = Intersection(name="intersection_1")
param = {"k": 3, "max_iter": 10}
hetero_kmeans_0 = HeteroKmeans(name='hetero_kmeans_0', **param)
hetero_kmeans_1 = HeteroKmeans(name='hetero_kmeans_1')
evaluation_0 = Evaluation(name='evaluation_0', eval_type='clustering')
evaluation_1 = Evaluation(name='evaluation_1', eval_type='clustering')
# add components to pipeline, in order of task execution
pipeline.add_component(reader_0)
pipeline.add_component(reader_1)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(data_transform_1,
data=Data(data=reader_1.output.data),
model=Model(data_transform_0.output.model))
# set data input sources of intersection components
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(intersection_1,
data=Data(data=data_transform_1.output.data))
# set train & validate data of hetero_lr_0 component
pipeline.add_component(hetero_kmeans_0,
data=Data(train_data=intersection_0.output.data))
pipeline.add_component(hetero_kmeans_1,
data=Data(train_data=intersection_1.output.data))
# print(f"data: {hetero_kmeans_0.output.data.data[0]}")
pipeline.add_component(evaluation_0,
data=Data(data=hetero_kmeans_0.output.data.data[0]))
pipeline.add_component(evaluation_1,
data=Data(data=hetero_kmeans_1.output.data.data[0]))
# compile pipeline once finished adding modules, this step will form conf and dsl files for running job
pipeline.compile()
# fit model
pipeline.fit()
# query component summary
print(pipeline.get_component("hetero_kmeans_0").get_summary())
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = [{"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"},
{"name": "motor_hetero_guest", "namespace": f"experiment{namespace}"}]
host_train_data = [{"name": "motor_hetero_host", "namespace": f"experiment{namespace}"},
{"name": "motor_hetero_host", "namespace": f"experiment{namespace}"}]
pipeline = PipeLine().set_initiator(role='guest', party_id=guest).set_roles(guest=guest, host=host, arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data[0])
reader_0.get_party_instance(role='host', party_id=host).component_param(table=host_train_data[0])
reader_1 = Reader(name="reader_1")
reader_1.get_party_instance(role='guest', party_id=guest).component_param(table=guest_train_data[1])
reader_1.get_party_instance(role='host', party_id=host).component_param(table=host_train_data[1])
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_1 = DataTransform(name="data_transform_1")
data_transform_0.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed",
label_type="float", output_format="dense")
data_transform_0.get_party_instance(role='host', party_id=host).component_param(with_label=False)
data_transform_1.get_party_instance(role='guest', party_id=guest).component_param(with_label=True, label_name="motor_speed",
label_type="float", output_format="dense")
data_transform_1.get_party_instance(role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
intersect_1 = Intersection(name="intersection_1")
hetero_linr_0 = HeteroLinR(name="hetero_linr_0", penalty="L2", optimizer="sgd", tol=0.001,
alpha=0.01, max_iter=20, early_stop="weight_diff", batch_size=-1,
learning_rate=0.15, decay=0.0, decay_sqrt=False,
init_param={"init_method": "zeros"},
encrypted_mode_calculator_param={"mode": "fast"},
callback_param={"callbacks": ["EarlyStopping", "PerformanceEvaluate"],
"validation_freqs": 1,
"early_stopping_rounds": 5,
"metrics": [
"mean_absolute_error",
"root_mean_squared_error"
],
"use_first_metric_only": False,
"save_freq": 1
}
)
pipeline.add_component(reader_0)
pipeline.add_component(reader_1)
pipeline.add_component(data_transform_0, data=Data(data=reader_0.output.data))
pipeline.add_component(data_transform_1, data=Data(data=reader_1.output.data), model=Model(data_transform_0.output.model))
pipeline.add_component(intersection_0, data=Data(data=data_transform_0.output.data))
pipeline.add_component(intersect_1, data=Data(data=data_transform_1.output.data))
pipeline.add_component(hetero_linr_0, data=Data(train_data=intersection_0.output.data,
validate_data=intersect_1.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "motor_hetero_mini_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "motor_hetero_mini_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(role='guest',
party_id=guest).component_param(
with_label=True,
output_format="dense",
label_name="motor_speed",
label_type="float",
)
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_linr_0 = HeteroLinR(name="hetero_linr_0",
early_stop="diff",
max_iter=3,
penalty="None",
optimizer="sgd",
tol=0.001,
alpha=0.01,
batch_size=-1,
learning_rate=0.15,
decay=0.0,
decay_sqrt=False,
init_param={"init_method": "zeros"},
stepwise_param={
"score_name": "AIC",
"direction": "backward",
"need_stepwise": True,
"max_step": 3,
"nvmin": 2
})
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_linr_0,
data=Data(train_data=intersection_0.output.data))
pipeline.compile()
pipeline.fit()
def main(config="../../config.yaml", namespace=""):
# obtain config
if isinstance(config, str):
config = load_job_config(config)
parties = config.parties
guest = parties.guest[0]
host = parties.host[0]
arbiter = parties.arbiter[0]
guest_train_data = {
"name": "motor_hetero_guest",
"namespace": f"experiment{namespace}"
}
host_train_data = {
"name": "motor_hetero_host",
"namespace": f"experiment{namespace}"
}
pipeline = PipeLine().set_initiator(
role='guest', party_id=guest).set_roles(guest=guest,
host=host,
arbiter=arbiter)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(
role='guest', party_id=guest).component_param(table=guest_train_data)
reader_0.get_party_instance(
role='host', party_id=host).component_param(table=host_train_data)
data_transform_0 = DataTransform(name="data_transform_0")
data_transform_0.get_party_instance(
role='guest', party_id=guest).component_param(with_label=True,
label_name="motor_speed",
label_type="float",
output_format="dense")
data_transform_0.get_party_instance(
role='host', party_id=host).component_param(with_label=False)
intersection_0 = Intersection(name="intersection_0")
hetero_linr_0 = HeteroLinR(
name="hetero_linr_0",
penalty="None",
optimizer="sgd",
tol=0.001,
alpha=0.01,
max_iter=20,
early_stop="weight_diff",
batch_size=-1,
learning_rate=0.15,
decay=0.0,
decay_sqrt=False,
init_param={"init_method": "zeros"},
encrypted_mode_calculator_param={"mode": "fast"},
cv_param={
"n_splits": 5,
"shuffle": False,
"random_seed": 42,
"need_cv": True
})
pipeline.add_component(reader_0)
pipeline.add_component(data_transform_0,
data=Data(data=reader_0.output.data))
pipeline.add_component(intersection_0,
data=Data(data=data_transform_0.output.data))
pipeline.add_component(hetero_linr_0,
data=Data(train_data=intersection_0.output.data))
pipeline.compile()
pipeline.fit()
