def initialization(*args):
"""
:param:tuple : (b_context, b_X, b_X,keys)
b_context: binary representation of the context_. context_.serialize()$
b_X : list of binary representations of samples from CKKS vectors format
b_Y : list of binary representations of labels from CKKS vectors format
keys : keys of the samples which are passed to the subprocess. the local b_X[i] is the global X[keys[i]]. Useful to map predictions to true labels
This function is the first one to be passed in the input_queue queue of the process.
It first deserialize the context_, passing it global,
in the memory space allocated to the process
Then the batch is also deserialize, using the context_,
to generate a list of CKKS vector which stand for the encrypted samples on which the process will work
"""
b_context = args[0]
b_X = args[1]
b_Y = args[2]
global context_
context_ = ts.context_from(b_context)
global local_X_
global local_Y_
local_X_ = [ts.ckks_vector_from(context_, i) for i in b_X]
local_Y_ = [ts.ckks_vector_from(context_, i) for i in b_Y]
global local_keys
local_keys = args[3]
return 'Initialization done for process %s. Len of data : %i' % (
multiprocessing.current_process().name, len(local_X_))
def load_dataset(
dataset_id: str,
) -> Tuple[
ts._ts_cpp.TenSEALContext,
List[ts._ts_cpp.CKKSVector],
List[ts._ts_cpp.CKKSVector],
int,
]:
"""Load a dataset into CKKSVectors"""
ctx_id, X, Y, batch_size = get_raw_dataset(dataset_id)
ctx = load_context(ctx_id)
enc_X = [ts.ckks_vector_from(ctx, x) for x in X]
enc_Y = [ts.ckks_vector_from(ctx, y) for y in Y]
return ctx, enc_X, enc_Y, batch_size
def get_dataset(
self, dataset_id: str, context: ts._ts_cpp.TenSEALContext
) -> Tuple[str, List[ts._ts_cpp.CKKSVector], int]:
"""Get a previously registered dataset using its id
Args:
dataset_id: id referring to the previously saved dataset
context: TenSEALContext used to load the dataset
Returns:
(context_id, X, Y, batch_size)
Raises:
ConnectionError: if a connection can't be established with the API
ResourceNotFound: if the dataset identified with `dataset_id` can't be found
Answer418: if response.status_code is 418
ServerError: if response.status_code is 500
"""
url = self._base_url + f"/datasets/"
data = {"dataset_id": dataset_id}
try:
response = requests.get(url, params=data)
except requests.exceptions.ConnectionError:
raise ConnectionError
if response.status_code != 200:
Client._handle_error_response(response)
resp_json = response.json()
ctx_id = resp_json["context_id"]
batch_size = resp_json["batch_size"]
enc_X, enc_Y = [], []
for x_buff, y_buff in zip(resp_json["X"], resp_json["Y"]):
x = ts.ckks_vector_from(context, b64decode(x_buff))
y = ts.ckks_vector_from(context, b64decode(y_buff))
enc_X.append(x)
enc_Y.append(y)
return ctx_id, enc_X, enc_Y, batch_size
def evaluate(
self,
model_name: str,
context: Union[ts._ts_cpp.TenSEALContext, bytes], # serialized or not
ckks_vector: Union[ts._ts_cpp.CKKSVector, bytes], # serialized or not
) -> ts._ts_cpp.CKKSVector:
"""Evaluate model `model_name` on the encrypted input data `ckks_vector`
Args:
model_name: the model name to use for evaluation
context: TenSEALContext containing keys needed for evaluation
ckks_vector: encrypted input to feed the model with
Returns:
tenseal.CKKSVector: encrypted output of the evaluation
Raises:
ConnectionError: if a connection can't be established with the API
Answer418: if response.status_code is 418
ServerError: if response.status_code is 500
"""
url = self._base_url + f"/eval/{model_name}"
# don't serialize if already
if not isinstance(context, bytes):
ser_ctx = context.serialize()
else:
ser_ctx = context
if not isinstance(ckks_vector, bytes):
ser_vec = ckks_vector.serialize()
else:
ser_vec = ckks_vector
data = {
"context": b64encode(ser_ctx).decode(),
"ckks_vector": b64encode(ser_vec).decode(),
}
try:
response = requests.post(url, json=data)
except requests.exceptions.ConnectionError:
raise ConnectionError
if response.status_code != 200:
Client._handle_error_response(response)
ser_result = response.json()["ckks_vector"]
result = ts.ckks_vector_from(context, b64decode(ser_result))
return result
def prepare_input(context, ckks_vector):
try:
ctx = ts.context_from(context)
enc_x = ts.ckks_vector_from(ctx, ckks_vector)
except:
raise DeserializationError(
"cannot deserialize context or ckks_vector")
# TODO: replace this with a more flexible check when introduced in the API
try:
_ = ctx.galois_keys()
except:
raise InvalidContext("the context doesn't hold galois keys")
return enc_x
def prepare_input(context: bytes, ckks_vector: bytes) -> ts._ts_cpp.CKKSVector:
# TODO: check parameters or size and raise InvalidParameters when needed
try:
ctx = ts.context_from(context)
enc_x = ts.ckks_vector_from(ctx, ckks_vector)
except:
raise DeserializationError("cannot deserialize context or ckks_vector")
# TODO: replace this with a more flexible check when introduced in the API
try:
_ = ctx.galois_keys()
except:
raise InvalidContext("the context doesn't hold galois keys")
return enc_x
def load_ctx_and_input(
context_file: typer.FileBinaryRead,
input_file: typer.FileBinaryRead = None
) -> Tuple[ts._ts_cpp.TenSEALContext, ts._ts_cpp.CKKSVector]:
try:
ctx = ts.context_from(context_file.read())
except Exception as e:
typer.echo(f"Couldn't load context: {str(e)}", err=True)
raise typer.Exit(code=1)
# only load context
if input_file is None:
return ctx, None
try:
enc_input = ts.ckks_vector_from(ctx, input_file.read())
except Exception as e:
typer.echo(f"Couldn't load encrypted input: {str(e)}", err=True)
raise typer.Exit(code=1)
return ctx, enc_input
def recreate_ckks(vec):
vec_proto = vec.serialize()
return ts.ckks_vector_from(vec.context(), vec_proto)
def forward_backward(*args):
"""
Compute the forward then backward propagation on the local part of the dataset (part of the process)
:param args: arg[0] serialized weight, arg[1] serialized bias
:return: serialized grad of weight.
serialized grad of bias.
serialized predictions on the local part of the dataset, computed by the forward propagation.
list of key, int which stand for the id of the data computed (useful for map the predictions)
"""
b_weight = args[0]
b_bias = args[1]
def sigmoid(enc_x, mult_coeff=1):
"""
Sigmoid implementation
We use the polynomial approximation of degree 3
sigmoid(x) = 0.5 + 0.197 * x - 0.004 * x^3
from https://eprint.iacr.org/2018/462.pdf
:param enc_x:
:param mult_coeff: The return is equivalent to sigmoid(x) * mult_coeff, but save one homomorph multiplication
:return: CKKS vector (result of sigmoid(x))
"""
poly_coeff = [0.5, 0.197, 0, -0.004]
return enc_x.polyval([i * mult_coeff for i in poly_coeff])
def forward(local_weight, local_bias, vec, mult_coeff=1):
"""
Compute forward propagation on a CKKS vector (or a list of CKKS vectors)
:param local_bias: encrypted bias used in forward propagation
:param local_weight: encrypted weight used in forward propagation
:param vec: CKKS vector or list of CKKS vector on which we want to make local_predictions (ie forward propagation
:param mult_coeff: The return is equivalent to forward(x) * mult_coeff, but save one homomorph multiplication
:return: encrypted prediction or list of encrypted local_predictions
"""
if type(vec) == list:
temp = [i.dot(local_weight) + local_bias for i in vec]
return [sigmoid(i, mult_coeff=mult_coeff) for i in temp]
else:
res = vec.dot(local_weight) + local_bias
return sigmoid(res, mult_coeff=mult_coeff)
def backward(X, local_predictions, Y):
"""
Compute the backpropagation on a given encrypted batch
:param X: list of encrypted (CKKS vectors). Features of the data on which the gradient will be computed (backpropagation)
:param local_predictions: list of encrypted CKKS vectors. Label predictions (forward propagation) on the data on which the gradient will be computed (backpropagation)
:param Y: list of encrypted CKKS vectors. Label of the data on which the gradient will be computed (backpropagation)
:return: Tuple of 2 CKKS vectors. Encrypted direction of descent for weight and bias"
"""
if type(X) == list:
err = local_predictions[0] - Y[0]
grad_weight = X[0] * err
grad_bias = err
for i in range(1, len(X)):
err = local_predictions[i] - Y[i]
grad_weight += X[i] * err
grad_bias += err
return grad_weight, grad_bias
else:
err = local_predictions - Y
grad_weight = X * err
grad_bias = err
return grad_weight, grad_bias
bias = ts.ckks_vector_from(context_, b_bias)
weight = ts.ckks_vector_from(context_, b_weight)
predictions = forward(local_bias=bias, local_weight=weight, vec=local_X_)
grads = backward(local_X_, predictions, local_Y_)
b_grad_weight = grads[0].serialize()
b_grad_bias = grads[1].serialize()
b_predictions = [prediction.serialize() for prediction in predictions]
return b_grad_weight, b_grad_bias, b_predictions, local_keys
def fit(self, X, Y):
"""
Train the model over encrypted data.
:param X: list of CKKS vectors: encrypted samples (train set)
:param Y: list of CKKS vectors: encrypted labels (train set)
"""
keys = [[] for _ in range(self.n_jobs)]
b_X = [[] for _ in range(self.n_jobs)]
b_Y = [[] for _ in range(self.n_jobs)]
self.logger.info("Starting serialization of data")
ser_time = time.time()
for i in range(len(X)):
b_X[i % self.n_jobs].append(X[i].serialize())
b_Y[i % self.n_jobs].append(Y[i].serialize())
keys[i % self.n_jobs].append(i)
self.logger.info("Data serialization done in %s seconds" %
(time.time() - ser_time))
inv_n = (1 / len(Y))
self.logger.info("Initialization of %d workers" % self.n_jobs)
init_work_timer = time.time()
list_queue_in = []
queue_out = multiprocessing.Queue()
init_tasks = [(initialization, (self.b_context, x, y, key))
for x, y, key in zip(b_X, b_Y, keys)]
for init in init_tasks:
list_queue_in.append(multiprocessing.Queue())
list_queue_in[-1].put(init)
multiprocessing.Process(target=worker,
args=(list_queue_in[-1],
queue_out)).start()
log_out = []
for _ in range(self.n_jobs):
log_out.append(queue_out.get())
logging.info(log_out[-1])
self.logger.info("Initialization done in %s seconds" %
(time.time() - init_work_timer))
del b_X
del b_Y
while self.iter < self.num_iter:
timer_iter = time.time()
self.weight = self.refresh(self.weight)
self.bias = self.refresh(self.bias)
b_weight = self.weight.serialize()
b_bias = self.bias.serialize()
for q in list_queue_in:
q.put((forward_backward, (
b_weight,
b_bias,
)))
direction_weight, direction_bias = 0, 0
b_predictions = [0 for _ in range(len(X))]
for _ in range(self.n_jobs):
child_process_ans = queue_out.get()
direction_weight += ts.ckks_vector_from(
self.context, child_process_ans[0])
direction_bias += ts.ckks_vector_from(self.context,
child_process_ans[1])
for pred, key in zip(child_process_ans[2],
child_process_ans[3]):
b_predictions[key] = pred
direction_weight = (direction_weight * self.lr *
inv_n) + (self.weight *
(self.lr * inv_n * self.reg_para))
direction_bias = direction_bias * self.lr * inv_n + (
self.bias * (self.lr * inv_n * self.reg_para))
self.weight -= direction_weight
self.bias -= direction_bias
self.logger.info(
"Just finished iteration number %d in %s seconds. " %
(self.iter, time.time() - timer_iter))
if self.verbose > 0 and self.iter % self.verbose == 0:
self.weight = self.refresh(self.weight)
self.bias = self.refresh(self.bias)
self.loss_list.append(
self.loss(Y, (ts.ckks_vector_from(self.context, prediction)
for prediction in b_predictions)))
self.logger.info("Starting computation of the loss ...")
self.logger.info('Loss : ' + str(self.loss_list[-1]) + ".")
if self.save_weight > 0 and self.iter % self.save_weight == 0:
self.weight_list.append(self.weight)
self.bias_list.append(self.bias)
self.iter += 1
for q in list_queue_in:
q.put('STOP')
self.weight = self.refresh(self.weight)
self.bias = self.refresh(self.bias)
return self
def client_deserialize(ctx, result):
if (not isinstance(result, list)):
return ckks_vector_from(ctx, b64decode(result))
else:
return [client_deserialize(ctx, e) for e in result]
def deserialize_decrypt(ctx, vector, secret_key):
if (isinstance(vector, list)):
return [deserialize_decrypt(ctx, e, secret_key) for e in vector]
else:
return ckks_vector_from(ctx, b64decode(vector)).decrypt(secret_key)
def unpack(self, data_file):
with open('{}.bin'.format(data_file), 'rb') as f:
data_serial = f.read()
return ts.ckks_vector_from(self.public_context, data_serial)
def receive_results(self, result_file, flag_file):
with open('{}.bin'.format(result_file), 'rb') as f:
res_serial = f.read()
with open('{}.bin'.format(flag_file), 'rb') as f:
flag = bool(f.read())
return flag, ts.ckks_vector_from(self.public_context, res_serial)
import tenseal as ts
import time
import p2p
bin_name = 'vectorReceive.bin'
context_name = 'contextReceive.bin'
receiver = p2p.Receiver('127.0.0.1', 8080)
receiver.receive(bin_name)
receiver = p2p.Receiver('127.0.0.1', 8080)
receiver.receive(context_name)
with open(bin_name, 'rb') as f:
vector_enc_bin = f.read()
with open(context_name, 'rb') as f:
context_bin = f.read()
context = ts.context_from(context_bin)
vector_enc = ts.ckks_vector_from(context, vector_enc_bin)
vector = vector_enc.decrypt()
print("Received vector:", vector)
