def iterator_to_weights(request_iterator: Iterator[WeightsPart],
decode=True) -> Weights:
first_weights_part = next(request_iterator)
full_weights = bytearray(first_weights_part.total_bytes)
bytes_sum = 0
training_summary = None
end_index = first_weights_part.byte_index + len(first_weights_part.weights)
full_weights[first_weights_part.
byte_index:end_index] = first_weights_part.weights
bytes_sum += len(first_weights_part.weights)
for weights_part in request_iterator:
end_index = weights_part.byte_index + len(weights_part.weights)
full_weights[weights_part.byte_index:end_index] = weights_part.weights
bytes_sum += len(weights_part.weights)
if weights_part.HasField('training_summary'):
training_summary = transform_training_summary_from_pb(
weights_part.training_summary)
weights_bytes = bytes(full_weights)
if decode:
return Weights(weights=decode_weights(weights_bytes),
training_summary=training_summary)
else:
# On the client side we can't necessarily unpickle the Weights object because the relevant libraries might not
# be importable. But we need to return a Weights object to match the MLI. So we wrap the pickled Weights in
# another Weights object.
return Weights(weights=weights_bytes,
training_summary=training_summary)
def mli_get_current_weights(self):
params = self.model.get_params()
return Weights(weights=dict(estimators_=self.model.estimators_,
n_classes_=self.model.n_classes_,
n_outputs_=self.model.n_outputs_,
classes_=self.model.classes_,
n_estimators=params["n_estimators"]))
def mli_get_current_weights(self):
"""
:return: The current weights of the model
"""
return Weights(weights=dict(coef_=self.model.coef_,
intercept_=self.model.intercept_))
def mli_propose_weights(self) -> Weights:
"""
Trains model on training set and returns new weights after training
- Current model is reverted to original state after training
:return: Weights after training
"""
current_weights = self.mli_get_current_weights()
if self.diff_priv_config is not None:
epsilon_after_training = self.get_privacy_budget()
if epsilon_after_training > self.diff_priv_budget.target_epsilon:
return Weights(
weights=current_weights,
training_summary=TrainingSummary(
dp_budget=self.diff_priv_budget,
error_code=ErrorCodes.DP_BUDGET_EXCEEDED
)
)
self.train()
new_weights = self.mli_get_current_weights()
self.set_weights(current_weights)
if self.diff_priv_config is not None:
self.diff_priv_budget.consumed_epsilon = epsilon_after_training
self.cumulative_epochs += self.epochs_per_proposal
new_weights.training_summary = TrainingSummary(dp_budget=self.diff_priv_budget)
return new_weights
def mli_get_current_weights(self) -> Weights:
"""
:return: The current weights of the model
"""
w = Weights(weights=[x.clone() for x in self.model.parameters()])
return w
def test_encode_decode():
test_weights = "weights"
weights = Weights(weights=test_weights)
encoded = encode_weights(weights)
decoded = decode_weights(encoded)
assert decoded == weights
assert weights.weights == test_weights
encoded2 = encode_weights(decoded)
assert encoded == encoded2
def test_iterator_and_back():
part_a = bytes(b"a" * WEIGHTS_PART_SIZE_BYTES)
part_b = bytes(b"b" * (WEIGHTS_PART_SIZE_BYTES - 2))
test_weights = part_a + part_b
weights = Weights(weights=test_weights)
iterator = weights_to_iterator(input_weights=weights, encode=False)
result = iterator_to_weights(request_iterator=iterator, decode=False)
assert result == weights
def mli_get_current_weights(self) -> Weights:
"""
:return: The current weights of the model
"""
current_state_dict = OrderedDict()
for key in self.model.state_dict():
current_state_dict[key] = self.model.state_dict()[key].clone()
w = Weights(
weights=current_state_dict, training_summary=self.get_training_summary()
)
return w
def test_weights_to_iterator_small():
part_a = bytes(b"a")
test_weights = part_a
weights = Weights(weights=test_weights)
iterator = weights_to_iterator(input_weights=weights, encode=False)
val = next(iterator, b"")
assert isinstance(val, WeightsPart)
assert val.total_bytes == 1
assert val.byte_index == 0
assert bytes(val.weights) == part_a
val = next(iterator, b"")
assert val == b""
def test_weights_to_iterator_small_limit():
part_a = bytes(b"a" * WEIGHTS_PART_SIZE_BYTES)
test_weights = part_a
weights = Weights(weights=test_weights)
iterator = weights_to_iterator(input_weights=weights, encode=False)
val = next(iterator, b"")
assert isinstance(val, WeightsPart)
assert val.total_bytes == WEIGHTS_PART_SIZE_BYTES
assert val.byte_index == 0
assert bytes(val.weights) == part_a
val = next(iterator, b"")
assert val == b""
def mli_get_current_weights(self) -> Weights:
return Weights(weights=self.current_value)
def mli_propose_weights(self):
self.current_value += 1
return Weights(weights=self.current_value)
def test_criterion(nkl):
nkl.criterion = "accuracy"
nkl.mli_accept_weights(Weights(weights="foo"))
assert nkl.vote_score == get_mock_model().evaluate.return_value["accuracy"]
def mli_get_current_weights(self):
return Weights(weights=pickle.dumps(self.model))
def mli_get_current_weights(self) -> Weights:
"""
:return: The current weights of the model
"""
return Weights(weights=self.model.get_weights())
def mli_get_current_weights(self):
return Weights(weights=dict(coef_=self.model.coef_,
intercept_=self.model.intercept_))
def mli_get_current_weights(self) -> Weights:
model_path = self.model_file_base + '_' + str(self.n_saves)
self.model.save_model(model_path)
self.n_saves += 1
return Weights(weights=model_path)
def test_accept_weights(nkl):
nkl.mli_accept_weights(Weights(weights=MODEL_PARAMETERS2))
assert str(nkl.model.parameters()) == str(MODEL_PARAMETERS2)
