def model_convert(model_contents: dict, module_name: str, framework_name=None):
"""Convert a H**o model component into format of a common ML framework
:param model_contents: The model dict un-serialized from the model protobuf.
:param module_name: The module name, typically as HomoXXXX.
:param framework_name: The wanted framework, e.g. "sklearn", "pytorch", etc.
If not specified, the target framework will be chosen
automatically.
:return: the converted framework name and a instance of the model object from
the specified framework.
"""
if not framework_name:
framework_name = get_default_target_framework(model_contents,
module_name)
if not framework_name:
return None, None
target_framework, component_converter = _get_component_converter(
module_name, framework_name)
if not component_converter:
LOGGER.warn(
f"Module {module_name} cannot be converted to framework {framework_name}"
)
return None, None
LOGGER.info(
f"Converting {module_name} module to a model of framework {target_framework}"
)
return target_framework, component_converter.convert(model_contents)
def client_fit(self, data_inst):
self._header = data_inst.schema["header"]
if not self.component_properties.is_warm_start:
client_align_labels(self, data_inst=data_inst)
data = self.data_converter.convert(
data_inst,
batch_size=self.batch_size,
encode_label=self.encode_label,
label_mapping=self._label_align_mapping,
)
if not self.component_properties.is_warm_start:
self.nn_model = self.model_builder(
input_shape=data.get_shape()[0],
nn_define=self.nn_define,
optimizer=self.optimizer,
loss=self.loss,
metrics=self.metrics,
)
else:
self.callback_warm_start_init_iter(self.aggregate_iteration_num + 1)
epoch_degree = float(len(data)) * self.aggregate_every_n_epoch
while self.aggregate_iteration_num + 1 < self.max_aggregate_iteration_num:
# update iteration num
self.aggregate_iteration_num += 1
self.callback_list.on_epoch_begin(self.aggregate_iteration_num)
LOGGER.info(f"start {self.aggregate_iteration_num}_th aggregation")
# train
self.nn_model.train(
data, aggregate_every_n_epoch=self.aggregate_every_n_epoch)
# send model for aggregate, then set aggregated model to local
self.aggregator.send_weighted_model(
weighted_model=self.nn_model.get_model_weights(),
weight=epoch_degree * self.aggregate_every_n_epoch,
suffix=_suffix(self),
)
weights = self.aggregator.get_aggregated_model(suffix=_suffix(self))
self.nn_model.set_model_weights(weights=weights)
self.callback_list.on_epoch_end(self.aggregate_iteration_num)
# calc loss and check convergence
if client_is_converged(self, data, epoch_degree):
LOGGER.info(f"early stop at iter {self.aggregate_iteration_num}")
break
LOGGER.info(
f"role {self.role} finish {self.aggregate_iteration_num}_th aggregation"
)
else:
LOGGER.warn(
f"reach max iter: {self.aggregate_iteration_num}, not converged")
self.set_summary(self._summary)
def fit(self, data_inst):
label_mapping = HomoLabelEncoderArbiter().label_alignment()
LOGGER.info(f'label mapping: {label_mapping}')
while self.aggregate_iteration_num < self.max_aggregate_iteration_num:
self.model = self.aggregator.weighted_mean_model(suffix=self._suffix())
self.aggregator.send_aggregated_model(model=self.model, suffix=self._suffix())
if self._is_converged():
LOGGER.info(f"early stop at iter {self.aggregate_iteration_num}")
break
self.aggregate_iteration_num += 1
else:
LOGGER.warn(f"reach max iter: {self.aggregate_iteration_num}, not converged")
self.set_summary(self._summary)
def server_fit(self, data_inst):
if not self.component_properties.is_warm_start:
label_mapping = HomoLabelEncoderArbiter().label_alignment()
LOGGER.info(f"label mapping: {label_mapping}")
else:
self.callback_warm_start_init_iter(self.aggregate_iteration_num + 1)
while self.aggregate_iteration_num + 1 < self.max_aggregate_iteration_num:
# update iteration num
self.aggregate_iteration_num += 1
self.callback_list.on_epoch_begin(self.aggregate_iteration_num)
self.model = self.aggregator.weighted_mean_model(suffix=_suffix(self))
self.aggregator.send_aggregated_model(model=self.model,
suffix=_suffix(self))
self.callback_list.on_epoch_end(self.aggregate_iteration_num)
if server_is_converged(self):
LOGGER.info(f"early stop at iter {self.aggregate_iteration_num}")
break
else:
LOGGER.warn(
f"reach max iter: {self.aggregate_iteration_num}, not converged")
self.set_summary(self._summary)
def fit(self, data_inst: CTableABC, *args):
self._header = data_inst.schema["header"]
self._align_labels(data_inst)
data = self.data_converter.convert(data_inst, batch_size=self.batch_size, encode_label=self.encode_label,
label_mapping=self._label_align_mapping)
self.nn_model = self.model_builder(input_shape=data.get_shape()[0],
nn_define=self.nn_define,
optimizer=self.optimizer,
loss=self.loss,
metrics=self.metrics)
epoch_degree = float(len(data)) * self.aggregate_every_n_epoch
while self.aggregate_iteration_num < self.max_aggregate_iteration_num:
LOGGER.info(f"start {self.aggregate_iteration_num}_th aggregation")
# train
self.nn_model.train(data, aggregate_every_n_epoch=self.aggregate_every_n_epoch)
# send model for aggregate, then set aggregated model to local
self.aggregator.send_weighted_model(weighted_model=self.nn_model.get_model_weights(),
weight=epoch_degree * self.aggregate_every_n_epoch,
suffix=self._suffix())
weights = self.aggregator.get_aggregated_model(suffix=self._suffix())
self.nn_model.set_model_weights(weights=weights)
# calc loss and check convergence
if self._is_converged(data, epoch_degree):
LOGGER.info(f"early stop at iter {self.aggregate_iteration_num}")
break
LOGGER.info(f"role {self.role} finish {self.aggregate_iteration_num}_th aggregation")
self.aggregate_iteration_num += 1
else:
LOGGER.warn(f"reach max iter: {self.aggregate_iteration_num}, not converged")
self.set_summary(self._summary)
