def _display_result(self, block_num=None):
if block_num is None:
self.callback_metric(metric_name=self.metric_name,
metric_namespace=self.metric_namespace,
metric_data=[
Metric("Coverage", self.coverage),
Metric("Block number", self.block_num)
])
self.tracker.set_metric_meta(
metric_namespace=self.metric_namespace,
metric_name=self.metric_name,
metric_meta=MetricMeta(self.metric_name,
metric_type="INTERSECTION"))
else:
self.callback_metric(metric_name=self.metric_name,
metric_namespace=self.metric_namespace,
metric_data=[
Metric("Coverage", self.coverage),
Metric("Block number", block_num)
])
self.tracker.set_metric_meta(
metric_namespace=self.metric_namespace,
metric_name=self.metric_name,
metric_meta=MetricMeta(self.metric_name,
metric_type="INTERSECTION"))
def fit(self, data):
self.__init_intersect_method()
if self.model_param.repeated_id_process:
if self.model_param.intersect_cache_param.use_cache is True and self.model_param.intersect_method == consts.RSA:
raise ValueError(
"Not support cache module while repeated id process.")
if len(
self.host_party_id_list
) > 1 and self.model_param.repeated_id_owner != consts.GUEST:
raise ValueError(
"While multi-host, repeated_id_owner should be guest.")
proc_obj = RepeatedIDIntersect(
repeated_id_owner=self.model_param.repeated_id_owner,
role=self.role)
data = proc_obj.run(data=data)
if self.model_param.allow_info_share:
if self.model_param.intersect_method == consts.RSA and self.model_param.info_owner == consts.GUEST \
or self.model_param.intersect_method == consts.RAW and self.model_param.join_role == self.model_param.info_owner:
self.model_param.sync_intersect_ids = False
self.intersect_ids = self.intersection_obj.run(data)
if self.model_param.allow_info_share:
self.intersect_ids = self.__share_info(data)
LOGGER.info("Finish intersection")
if self.intersect_ids:
self.intersect_num = self.intersect_ids.count()
self.intersect_rate = self.intersect_num * 1.0 / data.count()
self.set_summary(self.get_model_summary())
self.callback_metric(metric_name=self.metric_name,
metric_namespace=self.metric_namespace,
metric_data=[
Metric("intersect_count", self.intersect_num),
Metric("intersect_rate", self.intersect_rate)
])
self.tracker.set_metric_meta(metric_namespace=self.metric_namespace,
metric_name=self.metric_name,
metric_meta=MetricMeta(
name=self.metric_name,
metric_type=self.metric_type))
def callback(keyword="missing_impute",
value_list=None,
tracker=None):
# tracker = Tracking("abc", "123")
metric_type = None
"""
if keyword.endswith("ratio"):
metric_list = []
for i in range(len(value_list)):
metric_list.append(Metric(i, value_list[i]))
tracker.log_metric_data(keyword, "DATAIO", metric_list)
metric_type = "DATAIO_TABLE"
"""
metric_list = []
for i in range(len(value_list)):
metric_list.append(Metric(value_list[i], i))
tracker.log_metric_data(keyword, "DATAIO", metric_list)
metric_type = "DATAIO_TEXT"
tracker.set_metric_meta(keyword,
"DATAIO",
MetricMeta(name=keyword,
metric_type=metric_type))
def run(self, component_parameters=None, args=None):
self.parameters = component_parameters["DownloadParam"]
self.parameters["role"] = component_parameters["role"]
self.parameters["local"] = component_parameters["local"]
table_name, namespace = dtable_utils.get_table_info(config=self.parameters,
create=False)
job_id = self.taskid.split("_")[0]
session.init(job_id, self.parameters["work_mode"])
with open(os.path.abspath(self.parameters["output_path"]), "w") as fout:
data_table = session.get_data_table(name=table_name, namespace=namespace)
count = data_table.count()
LOGGER.info('===== begin to export data =====')
lines = 0
for key, value in data_table.collect():
if not value:
fout.write(key + "\n")
else:
fout.write(key + self.parameters.get("delimitor", ",") + value + "\n")
lines += 1
if lines % 2000 == 0:
LOGGER.info("===== export {} lines =====".format(lines))
if lines % 10000 == 0:
job_info = {'f_progress': lines/count*100//1}
self.update_job_status(self.parameters["local"]['role'], self.parameters["local"]['party_id'],
job_info)
self.update_job_status(self.parameters["local"]['role'],
self.parameters["local"]['party_id'], {'f_progress': 100})
self.callback_metric(metric_name='data_access',
metric_namespace='download',
metric_data=[Metric("count", data_table.count())])
LOGGER.info("===== export {} lines totally =====".format(lines))
LOGGER.info('===== export data finish =====')
LOGGER.info('===== export data file path:{} ====='.format(os.path.abspath(self.parameters["output_path"])))
def save_data_table(self, dst_table_name, dst_table_namespace, head=True, job_id=None):
input_file = self.parameters["file"]
count = self.get_count(input_file)
with open(input_file, 'r') as fin:
lines_count = 0
if head is True:
data_head = fin.readline()
count -= 1
self.save_data_header(data_head, dst_table_name, dst_table_namespace)
while True:
data = list()
lines = fin.readlines(self.MAX_BYTES)
if lines:
for line in lines:
values = line.replace("\n", "").replace("\t", ",").split(",")
data.append((values[0], self.list_to_str(values[1:])))
lines_count += len(data)
f_progress = lines_count/count*100//1
job_info = {'f_progress': f_progress}
self.update_job_status(self.parameters["local"]['role'], self.parameters["local"]['party_id'],
job_info)
data_table = session.save_data(data, name=dst_table_name, namespace=dst_table_namespace,
partition=self.parameters["partition"])
else:
self.tracker.save_data_view(role=self.parameters["local"]['role'],
party_id=self.parameters["local"]['party_id'],
data_info={'f_table_name': dst_table_name,
'f_table_namespace': dst_table_namespace,
'f_partition': self.parameters["partition"],
'f_table_create_count': data_table.count()
})
self.callback_metric(metric_name='data_access',
metric_namespace='upload',
metric_data=[Metric("count", data_table.count())])
return data_table.count()
def save_data_table(self,
dst_table_name,
dst_table_namespace,
head=True,
in_version=False):
input_file = self.parameters["file"]
count = self.get_count(input_file)
with open(input_file, 'r') as fin:
lines_count = 0
if head is True:
data_head = fin.readline()
count -= 1
self.save_data_header(data_head, dst_table_name,
dst_table_namespace)
self.table_info["cols"] = data_head
while True:
data = list()
lines = fin.readlines(self.MAX_BYTES)
if lines:
for line in lines:
values = line.replace("\n", "").replace("\t",
",").split(",")
data.append((values[0], self.list_to_str(values[1:])))
lines_count += len(data)
f_progress = lines_count / count * 100 // 1
job_info = {'f_progress': f_progress}
self.update_job_status(
self.parameters["local"]['role'],
self.parameters["local"]['party_id'], job_info)
data_table = session.save_data(
data,
name=dst_table_name,
namespace=dst_table_namespace,
partition=self.parameters["partition"])
self.table_info["v_len"] = data_table_count
else:
self.tracker.save_data_view(
role=self.parameters["local"]['role'],
party_id=self.parameters["local"]['party_id'],
data_info={
'f_table_name': dst_table_name,
'f_table_namespace': dst_table_namespace,
'f_partition': self.parameters["partition"],
'f_table_count_actual': data_table.count(),
'f_table_count_upload': count
})
self.callback_metric(
metric_name='data_access',
metric_namespace='upload',
metric_data=[Metric("count", data_table.count())])
if in_version:
version_log = "[AUTO] save data at %s." % datetime.datetime.now(
)
version_control.save_version(
name=dst_table_name,
namespace=dst_table_namespace,
version_log=version_log)
return data_table.count()
def fit(self, data_inst, validate_data=None):
validation_strategy = self.init_validation_strategy(
data_inst, validate_data)
self._build_model()
self.prepare_batch_data(self.batch_generator, data_inst)
if not self.input_shape:
self.model.set_empty()
self._set_loss_callback_info()
cur_epoch = 0
while cur_epoch < self.epochs:
LOGGER.debug("cur epoch is {}".format(cur_epoch))
epoch_loss = 0
for batch_idx in range(len(self.data_x)):
self.model.train(self.data_x[batch_idx],
self.data_y[batch_idx], cur_epoch, batch_idx)
self.reset_flowid()
metrics = self.model.evaluate(self.data_x[batch_idx],
self.data_y[batch_idx],
cur_epoch, batch_idx)
self.recovery_flowid()
LOGGER.debug("metrics is {}".format(metrics))
batch_loss = metrics["loss"]
epoch_loss += batch_loss
epoch_loss /= len(self.data_x)
LOGGER.debug("epoch {}' loss is {}".format(cur_epoch, epoch_loss))
self.callback_metric("loss", "train",
[Metric(cur_epoch, epoch_loss)])
self.history_loss.append(epoch_loss)
if validation_strategy:
validation_strategy.validate(self, cur_epoch)
is_converge = self.converge_func.is_converge(epoch_loss)
self.transfer_variable.is_converge.remote(is_converge,
role=consts.HOST,
idx=0,
suffix=(cur_epoch, ))
if is_converge:
LOGGER.debug(
"Training process is converged in epoch {}".format(
cur_epoch))
break
cur_epoch += 1
if cur_epoch == self.epochs:
LOGGER.debug(
"Training process reach max training epochs {} and not converged"
.format(self.epochs))
def __save_single_value(self, result, metric_name, metric_namespace,
eval_name):
self.tracker.log_metric_data(
metric_namespace, metric_name,
[Metric(eval_name, np.round(result, self.round_num))])
self.tracker.set_metric_meta(
metric_namespace, metric_name,
MetricMeta(name=metric_name, metric_type="EVALUATION_SUMMARY"))
def fit(self, data_inst, validate_data=None):
self.validation_strategy = self.init_validation_strategy(data_inst, validate_data)
self._build_model()
self.prepare_batch_data(self.batch_generator, data_inst)
if not self.input_shape:
self.model.set_empty()
self._set_loss_callback_info()
cur_epoch = 0
while cur_epoch < self.epochs:
LOGGER.debug("cur epoch is {}".format(cur_epoch))
epoch_loss = 0
for batch_idx in range(len(self.data_x)):
batch_loss = self.model.train(self.data_x[batch_idx], self.data_y[batch_idx], cur_epoch, batch_idx)
epoch_loss += batch_loss
epoch_loss /= len(self.data_x)
LOGGER.debug("epoch {}' loss is {}".format(cur_epoch, epoch_loss))
self.callback_metric("loss",
"train",
[Metric(cur_epoch, epoch_loss)])
self.history_loss.append(epoch_loss)
if self.validation_strategy:
self.validation_strategy.validate(self, cur_epoch)
if self.validation_strategy.need_stop():
LOGGER.debug('early stopping triggered')
break
if self.hetero_nn_param.selector_param.method:
# when use selective bp, loss converge will be disabled
is_converge = False
else:
is_converge = self.converge_func.is_converge(epoch_loss)
self._summary_buf["is_converged"] = is_converge
self.transfer_variable.is_converge.remote(is_converge,
role=consts.HOST,
idx=0,
suffix=(cur_epoch,))
if is_converge:
LOGGER.debug("Training process is converged in epoch {}".format(cur_epoch))
break
cur_epoch += 1
if cur_epoch == self.epochs:
LOGGER.debug("Training process reach max training epochs {} and not converged".format(self.epochs))
if self.validation_strategy and self.validation_strategy.has_saved_best_model():
self.load_model(self.validation_strategy.cur_best_model)
self.set_summary(self._get_model_summary())
def read_metric_data(self,
metric_namespace: str,
metric_name: str,
job_level=False):
metrics = []
for k, v in self.read_metrics_from_db(metric_namespace, metric_name, 1,
job_level):
metrics.append(Metric(key=k, value=v))
return metrics
def record_step_best(self, step_best, host_mask, guest_mask,
data_instances, model):
metas = {
"host_mask": host_mask.tolist(),
"guest_mask": guest_mask.tolist(),
"score_name": self.score_name
}
metas["number_in"] = int(sum(host_mask) + sum(guest_mask))
metas["direction"] = self.direction
metas["n_count"] = int(self.n_count)
host_party_id = model.component_properties.host_party_idlist[0]
guest_party_id = model.component_properties.guest_partyid
metas["host_features_anonym"] = [
f"host_{host_party_id}_{i}" for i in range(len(host_mask))
]
metas["guest_features_anonym"] = [
f"guest_{guest_party_id}_{i}" for i in range(len(guest_mask))
]
model_info = self.models_trained[step_best]
loss = model_info.get_loss()
ic_val = model_info.get_score()
metas["loss"] = loss
metas["current_ic_val"] = ic_val
metas["fit_intercept"] = model.fit_intercept
model_key = model_info.get_key()
model_dict = self._get_model(model_key)
if self.role != consts.ARBITER:
all_features = data_instances.schema.get('header')
metas["all_features"] = all_features
metas["to_enter"] = self.get_to_enter(host_mask, guest_mask,
all_features)
model_param = list(model_dict.get('model').values())[0].get(
model.model_param_name)
param_dict = MessageToDict(model_param)
metas["intercept"] = param_dict.get("intercept", None)
metas["weight"] = param_dict.get("weight", {})
metas["header"] = param_dict.get("header", [])
if self.n_step == 0 and self.direction == "forward":
metas["intercept"] = self.intercept
metric_name = f"stepwise_{self.n_step}"
metric = [Metric(metric_name, float(self.n_step))]
model.callback_metric(metric_name=metric_name,
metric_namespace=self.metric_namespace,
metric_data=metric)
model.tracker.set_metric_meta(metric_name=metric_name,
metric_namespace=self.metric_namespace,
metric_meta=MetricMeta(
name=metric_name,
metric_type=self.metric_type,
extra_metas=metas))
LOGGER.info(f"metric_name: {metric_name}, metas: {metas}")
return
def __save_single_value(self, result, metric_name, metric_namespace, eval_name):
metric_type = 'EVALUATION_SUMMARY'
if eval_name in consts.ALL_CLUSTER_METRICS:
metric_type = 'CLUSTERING_EVALUATION_SUMMARY'
self.tracker.log_metric_data(metric_namespace, metric_name,
[Metric(eval_name, np.round(result, self.round_num))])
self.tracker.set_metric_meta(metric_namespace, metric_name,
MetricMeta(name=metric_name, metric_type=metric_type))
def __save_curve_data(self, x_axis_list, y_axis_list, metric_name, metric_namespace):
points = []
for i, value in enumerate(x_axis_list):
if isinstance(value, float):
value = np.round(value, self.round_num)
points.append((value, np.round(y_axis_list[i], self.round_num)))
points.sort(key=lambda x: x[0])
metric_points = [Metric(point[0], point[1]) for point in points]
self.tracker.log_metric_data(metric_namespace, metric_name, metric_points)
def callback(self, metas):
metric = [Metric(self.metric_name, 0)]
self.callback_metric(metric_name=self.metric_name,
metric_namespace=self.metric_namespace,
metric_data=metric)
self.tracker.set_metric_meta(metric_name=self.metric_name,
metric_namespace=self.metric_namespace,
metric_meta=MetricMeta(
name=self.metric_name,
metric_type=self.metric_type,
extra_metas=metas))
def callback_loss(self, iter_num, loss):
metric_meta = MetricMeta(name='train',
metric_type=MetricType.LOSS,
extra_metas={
"unit_name": "iters",
})
self.callback_meta(metric_name='loss', metric_namespace='train', metric_meta=metric_meta)
self.callback_metric(metric_name='loss',
metric_namespace='train',
metric_data=[Metric(iter_num, loss)])
def fit(self, data):
self.__init_intersect_method()
self.intersect_ids = self.intersection_obj.run(data)
LOGGER.info("Finish intersection")
if self.intersect_ids:
self.intersect_num = self.intersect_ids.count()
self.intersect_rate = self.intersect_num * 1.0 / data.count()
self.callback_metric(metric_name=self.metric_name,
metric_namespace=self.metric_namespace,
metric_data=[
Metric("intersect_count", self.intersect_num),
Metric("intersect_rate", self.intersect_rate)
])
self.tracker.set_metric_meta(metric_namespace=self.metric_namespace,
metric_name=self.metric_name,
metric_meta=MetricMeta(
name=self.metric_name,
metric_type=self.metric_type))
def save_meta(self, dst_table_namespace, dst_table_name, table_count):
self.tracker.log_output_data_info(data_name='upload',
table_namespace=dst_table_namespace,
table_name=dst_table_name)
self.tracker.log_metric_data(metric_namespace="upload",
metric_name="data_access",
metrics=[Metric("count", table_count)])
self.tracker.set_metric_meta(metric_namespace="upload",
metric_name="data_access",
metric_meta=MetricMeta(
name='upload', metric_type='UPLOAD'))
def fit(self, data_inst, valid_inst=None):
self.federated_binning()
# initializing
self.feature_num = self.sync_feature_num()
self.tree_dim = 1
if self.task_type == consts.CLASSIFICATION:
label_mapping = self.label_alignment()
LOGGER.debug('label mapping is {}'.format(label_mapping))
self.tree_dim = len(label_mapping) if len(label_mapping) > 2 else 1
if self.n_iter_no_change:
self.check_convergence_func = converge_func_factory(
"diff", self.tol)
LOGGER.debug('begin to fit a boosting tree')
for epoch_idx in range(self.num_trees):
for t_idx in range(self.tree_dim):
valid_feature = self.sample_valid_feature()
self.send_valid_features(valid_feature, epoch_idx, t_idx)
flow_id = self.generate_flowid(epoch_idx, t_idx)
new_tree = HomoDecisionTreeArbiter(self.tree_param,
valid_feature=valid_feature,
epoch_idx=epoch_idx,
flow_id=flow_id,
tree_idx=t_idx)
new_tree.fit()
global_loss = self.aggregator.aggregate_loss(suffix=(epoch_idx, ))
self.global_loss_history.append(global_loss)
LOGGER.debug('cur epoch global loss is {}'.format(global_loss))
self.callback_metric("loss", "train",
[Metric(epoch_idx, global_loss)])
if self.n_iter_no_change:
should_stop = self.aggregator.broadcast_converge_status(
self.check_convergence, (global_loss, ),
suffix=(epoch_idx, ))
LOGGER.debug('stop flag sent')
if should_stop:
break
self.callback_meta(
"loss", "train",
MetricMeta(name="train",
metric_type="LOSS",
extra_metas={"Best": min(self.global_loss_history)}))
LOGGER.debug('fitting h**o decision tree done')
def callback_dbi(self, iter_num, dbi):
metric_meta = MetricMeta(name='train',
metric_type="DBI",
extra_metas={
"unit_name": "iters",
})
self.callback_meta(metric_name='DBI',
metric_namespace='train',
metric_meta=metric_meta)
self.callback_metric(metric_name='DBI',
metric_namespace='train',
metric_data=[Metric(iter_num, dbi)])
def fit(self, data_inst, validate_data=None):
# init aggregator
self.aggregator = HomoBoostArbiterAggregator()
self.binning_obj = HomoFeatureBinningServer()
self.federated_binning()
# initializing
self.feature_num = self.sync_feature_num()
if self.task_type == consts.CLASSIFICATION:
label_mapping = HomoLabelEncoderArbiter().label_alignment()
LOGGER.info('label mapping is {}'.format(label_mapping))
self.booster_dim = len(
label_mapping) if len(label_mapping) > 2 else 1
if self.n_iter_no_change:
self.check_convergence_func = converge_func_factory(
"diff", self.tol)
LOGGER.info('begin to fit a boosting tree')
for epoch_idx in range(self.boosting_round):
LOGGER.info('cur epoch idx is {}'.format(epoch_idx))
for class_idx in range(self.booster_dim):
model = self.fit_a_booster(epoch_idx, class_idx)
global_loss = self.aggregator.aggregate_loss(suffix=(epoch_idx, ))
self.history_loss.append(global_loss)
LOGGER.debug('cur epoch global loss is {}'.format(global_loss))
self.callback_metric("loss", "train",
[Metric(epoch_idx, global_loss)])
if self.n_iter_no_change:
should_stop = self.aggregator.broadcast_converge_status(
self.check_convergence, (global_loss, ),
suffix=(epoch_idx, ))
LOGGER.debug('stop flag sent')
if should_stop:
break
self.callback_meta(
"loss", "train",
MetricMeta(name="train",
metric_type="LOSS",
extra_metas={"Best": min(self.history_loss)}))
self.set_summary(self.generate_summary())
def callback_loss(self, iter_num, loss):
# noinspection PyTypeChecker
metric_meta = MetricMeta(name='train',
metric_type="LOSS",
extra_metas={
"unit_name": "iters",
})
self.callback_meta(metric_name='loss', metric_namespace='train', metric_meta=metric_meta)
self.callback_metric(metric_name='loss',
metric_namespace='train',
metric_data=[Metric(iter_num, loss)])
self._summary["loss_history"].append(loss)
def run(self, component_parameters=None, args=None):
self.parameters = component_parameters["DownloadParam"]
self.parameters["role"] = component_parameters["role"]
self.parameters["local"] = component_parameters["local"]
name, namespace = self.parameters.get("name"), self.parameters.get(
"namespace")
with open(os.path.abspath(self.parameters["output_path"]),
"w") as fout:
with storage.Session.build(
session_id=job_utils.generate_session_id(
self.tracker.task_id,
self.tracker.task_version,
self.tracker.role,
self.tracker.party_id,
suffix="storage",
random_end=True),
name=name,
namespace=namespace) as storage_session:
data_table = storage_session.get_table()
count = data_table.count()
LOGGER.info('===== begin to export data =====')
lines = 0
job_info = {}
job_info["job_id"] = self.tracker.job_id
job_info["role"] = self.tracker.role
job_info["party_id"] = self.tracker.party_id
for key, value in data_table.collect():
if not value:
fout.write(key + "\n")
else:
fout.write(key +
self.parameters.get("delimiter", ",") +
str(value) + "\n")
lines += 1
if lines % 2000 == 0:
LOGGER.info(
"===== export {} lines =====".format(lines))
if lines % 10000 == 0:
job_info["progress"] = lines / count * 100 // 1
ControllerClient.update_job(job_info=job_info)
job_info["progress"] = 100
ControllerClient.update_job(job_info=job_info)
self.callback_metric(
metric_name='data_access',
metric_namespace='download',
metric_data=[Metric("count", data_table.count())])
LOGGER.info("===== export {} lines totally =====".format(lines))
LOGGER.info('===== export data finish =====')
LOGGER.info('===== export data file path:{} ====='.format(
os.path.abspath(self.parameters["output_path"])))
def callback_loss(self, iter_num, loss):
# noinspection PyTypeChecker
metric_meta = MetricMeta(
name="train",
metric_type="LOSS",
extra_metas={
"unit_name": "iters",
},
)
self.callback_meta(metric_name="loss",
metric_namespace="train",
metric_meta=metric_meta)
self.callback_metric(
metric_name="loss",
metric_namespace="train",
metric_data=[Metric(iter_num, loss)],
)
def callback_loss(self, iter_num, loss):
"""
call back function of loss and metrics.
:param iter_num: iter number.
:param loss: loss type.
:return:
"""
metric_meta = MetricMeta(name='train',
metric_type="LOSS",
extra_metas={
"unit_name": "iters",
})
self.callback_meta(metric_name='loss',
metric_namespace='train',
metric_meta=metric_meta)
self.callback_metric(metric_name='loss',
metric_namespace='train',
metric_data=[Metric(iter_num, loss)])
def callback_info(self):
class_weight = None
classes = None
if self.class_weight:
class_weight = {str(k): v for k, v in self.class_weight.items()}
classes = sorted([str(k) for k in self.class_weight.keys()])
# LOGGER.debug(f"callback class weight is: {class_weight}")
metric_meta = MetricMeta(name='train',
metric_type=self.metric_type,
extra_metas={
"weight_mode": self.weight_mode,
"class_weight": class_weight,
"classes": classes,
"sample_weight_name":
self.sample_weight_name
})
self.callback_metric(metric_name=self.metric_name,
metric_namespace=self.metric_namespace,
metric_data=[Metric(self.metric_name, 0)])
self.tracker.set_metric_meta(metric_namespace=self.metric_namespace,
metric_name=self.metric_name,
metric_meta=metric_meta)
def fit(self, data_inst, validate_data=None):
LOGGER.info('begin to fit a hetero boosting model, model is {}'.format(
self.model_name))
self.data_inst = data_inst
self.data_bin, self.bin_split_points, self.bin_sparse_points = self.prepare_data(
data_inst)
self.y = self.get_label(self.data_bin)
self.classes_, self.num_classes, self.booster_dim = self.check_label()
LOGGER.info('class index is {}'.format(self.classes_))
self.loss = self.get_loss_function()
self.sync_booster_dim()
self.y_hat, self.init_score = self.get_init_score(
self.y, self.num_classes)
self.generate_encrypter()
self.callback_meta(
"loss", "train",
MetricMeta(name="train",
metric_type="LOSS",
extra_metas={"unit_name": "iters"}))
self.validation_strategy = self.init_validation_strategy(
data_inst, validate_data)
for epoch_idx in range(self.boosting_round):
LOGGER.info('cur epoch idx is {}'.format(epoch_idx))
for class_idx in range(self.booster_dim):
# fit a booster
model = self.fit_a_booster(epoch_idx, class_idx)
booster_meta, booster_param = model.get_model()
if booster_meta is not None and booster_param is not None:
self.booster_meta = booster_meta
self.boosting_model_list.append(booster_param)
# update predict score
cur_sample_weights = model.get_sample_weights()
self.y_hat = self.get_new_predict_score(self.y_hat,
cur_sample_weights,
dim=class_idx)
# compute loss
loss = self.compute_loss(self.y_hat, self.y)
self.history_loss.append(loss)
LOGGER.info("round {} loss is {}".format(epoch_idx, loss))
self.callback_metric("loss", "train", [Metric(epoch_idx, loss)])
if self.validation_strategy:
self.validation_strategy.validate(
self,
epoch_idx,
use_precomputed_train=True,
train_scores=self.score_to_predict_result(
data_inst, self.y_hat))
should_stop_a, should_stop_b = False, False
if self.validation_strategy is not None:
if self.validation_strategy.need_stop():
should_stop_a = True
if self.n_iter_no_change and self.check_convergence(loss):
should_stop_b = True
self.is_converged = True
self.sync_stop_flag(self.is_converged, epoch_idx)
if should_stop_a or should_stop_b:
break
self.callback_meta(
"loss", "train",
MetricMeta(name="train",
metric_type="LOSS",
extra_metas={"Best": min(self.history_loss)}))
if self.validation_strategy and self.validation_strategy.has_saved_best_model(
):
LOGGER.info('best model exported')
self.load_model(self.validation_strategy.cur_best_model)
# get summary
self.set_summary(self.generate_summary())
def fit(self, data_instances):
if not self.need_run:
return data_instances
self.init_schema(data_instances)
LOGGER.debug("Before trainning, self.header: {}".format(self.header))
self._abnormal_detection(data_instances)
self.__init_parameters(data_instances)
w = self.__init_model(data_instances)
for iter_num in range(self.max_iter):
# mini-batch
LOGGER.debug("In iter: {}".format(iter_num))
batch_data_generator = self.mini_batch_obj.mini_batch_data_generator(
)
batch_num = 0
total_loss = 0
for batch_data in batch_data_generator:
f = functools.partial(self.gradient_operator.compute,
coef=self.coef_,
intercept=self.intercept_,
fit_intercept=self.fit_intercept)
grad_loss = batch_data.mapPartitions(f)
n = batch_data.count()
if not self.use_encrypt:
grad, loss = grad_loss.reduce(
self.aggregator.aggregate_grad_loss)
grad = np.array(grad)
grad /= n
loss /= n
if self.updater is not None:
loss_norm = self.updater.loss_norm(self.coef_)
total_loss += loss + loss_norm
# if not self.use_loss:
# total_loss = np.linalg.norm(self.coef_)
if not self.need_one_vs_rest:
metric_meta = MetricMeta(
name='train',
metric_type="LOSS",
extra_metas={"unit_name": "iters"})
metric_name = self.get_metric_name('loss')
self.callback_meta(metric_name=metric_name,
metric_namespace='train',
metric_meta=metric_meta)
self.callback_metric(
metric_name=metric_name,
metric_namespace='train',
metric_data=[Metric(iter_num, total_loss)])
else:
grad, _ = grad_loss.reduce(self.aggregator.aggregate_grad)
grad = np.array(grad)
grad /= n
self.update_model(grad)
w = self.merge_model()
batch_num += 1
if self.use_encrypt and batch_num % self.re_encrypt_batches == 0:
to_encrypt_model_id = self.transfer_variable.generate_transferid(
self.transfer_variable.to_encrypt_model, iter_num,
batch_num)
federation.remote(
w,
name=self.transfer_variable.to_encrypt_model.name,
tag=to_encrypt_model_id,
role=consts.ARBITER,
idx=0)
re_encrypted_model_id = self.transfer_variable.generate_transferid(
self.transfer_variable.re_encrypted_model, iter_num,
batch_num)
LOGGER.debug("re_encrypted_model_id: {}".format(
re_encrypted_model_id))
w = federation.get(
name=self.transfer_variable.re_encrypted_model.name,
tag=re_encrypted_model_id,
idx=0)
w = np.array(w)
self.set_coef_(w)
model_transfer_id = self.transfer_variable.generate_transferid(
self.transfer_variable.host_model, iter_num)
federation.remote(w,
name=self.transfer_variable.host_model.name,
tag=model_transfer_id,
role=consts.ARBITER,
idx=0)
if not self.use_encrypt:
loss_transfer_id = self.transfer_variable.generate_transferid(
self.transfer_variable.host_loss, iter_num)
federation.remote(total_loss,
name=self.transfer_variable.host_loss.name,
tag=loss_transfer_id,
role=consts.ARBITER,
idx=0)
LOGGER.debug("model and loss sent")
final_model_id = self.transfer_variable.generate_transferid(
self.transfer_variable.final_model, iter_num)
w = federation.get(name=self.transfer_variable.final_model.name,
tag=final_model_id,
idx=0)
w = np.array(w)
self.set_coef_(w)
converge_flag_id = self.transfer_variable.generate_transferid(
self.transfer_variable.converge_flag, iter_num)
converge_flag = federation.get(
name=self.transfer_variable.converge_flag.name,
tag=converge_flag_id,
idx=0)
self.n_iter_ = iter_num
LOGGER.debug("converge_flag: {}".format(converge_flag))
if converge_flag:
break
def fit(self, data):
LOGGER.debug(f"fit receives data is {data}")
if not isinstance(data, dict):
raise ValueError(
"Union module must receive more than one table as input.")
empty_count = 0
combined_table = None
combined_schema = None
metrics = []
for (key, local_table) in data.items():
LOGGER.debug("table to combine name: {}".format(key))
num_data = local_table.count()
LOGGER.debug("table count: {}".format(num_data))
metrics.append(Metric(key, num_data))
self.add_summary(key, num_data)
if num_data == 0:
LOGGER.warning("Table {} is empty.".format(key))
if combined_table is None:
combined_table = local_table
combined_schema = local_table.schema
empty_count += 1
continue
local_is_data_instance = self.check_is_data_instance(local_table)
if combined_table is None:
self.is_data_instance = local_is_data_instance
LOGGER.debug(f"self.is_data_instance is {self.is_data_instance}, "
f"local_is_data_instance is {local_is_data_instance}")
if self.is_data_instance != local_is_data_instance:
raise ValueError(
f"Cannot combine DataInstance and non-DataInstance object. Union aborted."
)
if self.is_data_instance:
self.is_empty_feature = data_overview.is_empty_feature(
local_table)
if self.is_empty_feature:
LOGGER.warning("Table {} has empty feature.".format(key))
else:
self.check_schema_content(local_table.schema)
if combined_table is None:
# first table to combine
combined_table = local_table
combined_schema = local_table.schema
else:
self.check_id(local_table, combined_table)
self.check_label_name(local_table, combined_table)
self.check_header(local_table, combined_table)
if self.keep_duplicate:
repeated_ids = combined_table.join(local_table,
lambda v1, v2: 1)
self.repeated_ids = [v[0] for v in repeated_ids.collect()]
self.key = key
local_table = local_table.flatMap(self._renew_id)
combined_table = combined_table.union(local_table,
self._keep_first)
combined_table.schema = combined_schema
# only check feature length if not empty
if self.is_data_instance and not self.is_empty_feature:
self.feature_count = len(combined_schema.get("header"))
LOGGER.debug("feature count: {}".format(self.feature_count))
combined_table.mapValues(self.check_feature_length)
if combined_table is None:
num_data = 0
LOGGER.warning(
"All tables provided are empty or have empty features.")
else:
num_data = combined_table.count()
metrics.append(Metric("Total", num_data))
self.add_summary("Total", num_data)
self.callback_metric(metric_name=self.metric_name,
metric_namespace=self.metric_namespace,
metric_data=metrics)
self.tracker.set_metric_meta(metric_namespace=self.metric_namespace,
metric_name=self.metric_name,
metric_meta=MetricMeta(
name=self.metric_name,
metric_type=self.metric_type))
LOGGER.info(
"Union operation finished. Total {} empty tables encountered.".
format(empty_count))
return combined_table
def save_data_table(self,
job_id,
dst_table_name,
dst_table_namespace,
head=True):
input_file = self.parameters["file"]
input_feature_count = self.get_count(input_file)
with open(input_file, 'r') as fin:
lines_count = 0
if head is True:
data_head = fin.readline()
input_feature_count -= 1
self.table.get_meta().update_metas(
schema=data_utils.get_header_schema(
header_line=data_head,
id_delimiter=self.parameters["id_delimiter"]))
n = 0
while True:
data = list()
lines = fin.readlines(self.MAX_BYTES)
if lines:
for line in lines:
values = line.rstrip().split(
self.parameters["id_delimiter"])
data.append((
values[0],
data_utils.list_to_str(
values[1:],
id_delimiter=self.parameters["id_delimiter"])))
lines_count += len(data)
save_progress = lines_count / input_feature_count * 100 // 1
job_info = {
'progress': save_progress,
"job_id": job_id,
"role": self.parameters["local"]['role'],
"party_id": self.parameters["local"]['party_id']
}
ControllerClient.update_job(job_info=job_info)
self.table.put_all(data)
if n == 0:
self.table.get_meta().update_metas(part_of_data=data)
else:
table_count = self.table.count()
self.table.get_meta().update_metas(
count=table_count,
partitions=self.parameters["partition"])
self.tracker.log_output_data_info(
data_name='upload',
table_namespace=dst_table_namespace,
table_name=dst_table_name)
self.tracker.log_metric_data(
metric_namespace="upload",
metric_name="data_access",
metrics=[Metric("count", table_count)])
self.tracker.set_metric_meta(metric_namespace="upload",
metric_name="data_access",
metric_meta=MetricMeta(
name='upload',
metric_type='UPLOAD'))
return table_count
n += 1
def fit(self, data_inst, validate_data=None):
LOGGER.info("begin to train secureboosting guest model")
self.gen_feature_fid_mapping(data_inst.schema)
data_inst = self.data_alignment(data_inst)
self.convert_feature_to_bin(data_inst)
self.set_y()
self.update_f_value()
self.generate_encrypter()
self.sync_tree_dim()
self.callback_meta(
"loss", "train",
MetricMeta(name="train",
metric_type="LOSS",
extra_metas={"unit_name": "iters"}))
validation_strategy = self.init_validation_strategy(
data_inst, validate_data)
for i in range(self.num_trees):
self.compute_grad_and_hess()
for tidx in range(self.tree_dim):
tree_inst = HeteroDecisionTreeGuest(self.tree_param)
tree_inst.set_inputinfo(self.data_bin,
self.get_grad_and_hess(tidx),
self.bin_split_points,
self.bin_sparse_points)
valid_features = self.sample_valid_features()
tree_inst.set_valid_features(valid_features)
tree_inst.set_encrypter(self.encrypter)
tree_inst.set_encrypted_mode_calculator(
self.encrypted_calculator)
tree_inst.set_flowid(self.generate_flowid(i, tidx))
tree_inst.set_host_party_idlist(
self.component_properties.host_party_idlist)
tree_inst.set_runtime_idx(
self.component_properties.local_partyid)
tree_inst.fit()
tree_meta, tree_param = tree_inst.get_model()
self.trees_.append(tree_param)
if self.tree_meta is None:
self.tree_meta = tree_meta
self.update_f_value(new_f=tree_inst.predict_weights, tidx=tidx)
self.update_feature_importance(
tree_inst.get_feature_importance())
loss = self.compute_loss()
self.history_loss.append(loss)
LOGGER.info("round {} loss is {}".format(i, loss))
LOGGER.debug("type of loss is {}".format(type(loss).__name__))
self.callback_metric("loss", "train", [Metric(i, loss)])
if validation_strategy:
validation_strategy.validate(self, i)
if self.n_iter_no_change is True:
if self.check_convergence(loss):
self.sync_stop_flag(True, i)
break
else:
self.sync_stop_flag(False, i)
LOGGER.debug("history loss is {}".format(min(self.history_loss)))
self.callback_meta(
"loss", "train",
MetricMeta(name="train",
metric_type="LOSS",
extra_metas={"Best": min(self.history_loss)}))
LOGGER.info("end to train secureboosting guest model")
