class FateScript(FederationRuntime):
def __init__(self, fed_obj, _host, _port):
super().__init__(fed_obj.job_id, fed_obj.party_id, fed_obj.role,
fed_obj.runtime_conf, _host, _port)
self.trans_conf = file_utils.load_json_conf(
'contrib/fate_script/conf/FateScriptTransferVar.json')
self.encrypt_operator = None
def remote(self, obj, name: str, tag: str, role=None, idx=-1):
if isinstance(obj, Tensor):
super().remote(obj.store, name, tag, role, idx)
#print("inside remote obj:{}".format(obj.store.count()))
else:
super().remote(obj, name, tag, role, idx)
def get(self, name, tag, idx=-1):
obj = copy.deepcopy(super().get(name, tag, idx))
if isinstance(obj, _DTable):
return copy.deepcopy(TensorInEgg(self.encrypt_operator, None, obj))
elif isinstance(obj, np.ndarray):
return copy.deepcopy(TensorInPy(self.encrypt_operator, None, obj))
else:
return copy.deepcopy(obj)
def init_encrypt_operator(self):
self.encrypt_operator = PaillierEncrypt()
def init_public_key(self, key_length=1024):
self.encrypt_operator.generate_key(key_length)
return self.encrypt_operator.get_public_key()
def get_public_key(self, public_key):
self.encrypt_operator.set_public_key(public_key)
def run_with_num_hosts(self, num_hosts):
ratio = 0.3
key_size = 1024
import random
from federatedml.secureprotol.encrypt import PaillierEncrypt
PaillierEncrypt().generate_key(key_size)
cipher_dict = {}
for i in range(num_hosts):
if random.random() > ratio:
cipher = PaillierEncrypt()
cipher.generate_key(key_size)
cipher_dict[i] = cipher
else:
cipher_dict[i] = None
from federatedml.framework.weights import ListWeights
variables = ListWeights([random.random() for _ in range(10)])
arbiter, guest, *hosts = self.run_results(num_hosts, cipher_dict,
variables)
guest = guest.unboxed
hosts = [host.unboxed for host in hosts]
self.assertListEqual(guest, variables.unboxed)
host_decrypted = [
cipher_dict[i].decrypt_list(hosts[i])
if cipher_dict[i] else hosts[i] for i in range(num_hosts)
]
for i in range(len(guest)):
for j in range(num_hosts):
self.assertAlmostEqual(guest[i], host_decrypted[j][i])
def run_with_num_hosts(self, num_hosts):
ratio = 0.3
key_size = 1024
import random
from federatedml.secureprotol.encrypt import PaillierEncrypt
PaillierEncrypt().generate_key(key_size)
cipher_dict = {}
for i in range(num_hosts):
if random.random() > ratio:
cipher = PaillierEncrypt()
cipher.generate_key(key_size)
cipher_dict[i] = cipher
else:
cipher_dict[i] = None
arbiter, guest, *hosts = self.run_results(num_hosts, cipher_dict)
arbiter = [x[0].unboxed for x in arbiter]
guest = guest.unboxed
hosts = [hosts[i].weights.decrypted(cipher_dict[i]).unboxed if cipher_dict[i] else hosts[i].unboxed
for i in range(num_hosts)]
self.assertListEqual(arbiter[0], guest)
for i in range(len(hosts)):
self.assertListEqual(arbiter[i+1], hosts[i])
def gen_paillier_cipher_operator(self, transfer_variables, suffix=tuple()):
self._pubkey_transfer = transfer_variables.paillier_pubkey
cipher = PaillierEncrypt()
cipher.generate_key()
pub_key = cipher.get_public_key()
self._pubkey_transfer.remote(obj=pub_key,
role=consts.HOST,
idx=-1,
suffix=suffix)
return cipher
def paillier_keygen(self, key_length, suffix=tuple()):
cipher = PaillierEncrypt()
cipher.generate_key(key_length)
pub_key = cipher.get_public_key()
self._pubkey_transfer.remote(obj=pub_key,
role=consts.HOST,
idx=-1,
suffix=suffix)
self._pubkey_transfer.remote(obj=pub_key,
role=consts.GUEST,
idx=-1,
suffix=suffix)
return cipher
def paillier_keygen(self, key_length, suffix=tuple()) -> dict:
hosts_use_cipher = self._use_encrypt_transfer.get(suffix=suffix)
host_ciphers = dict()
for idx, use_encryption in enumerate(hosts_use_cipher):
if not use_encryption:
host_ciphers[idx] = None
else:
cipher = PaillierEncrypt()
cipher.generate_key(key_length)
pub_key = cipher.get_public_key()
self._pubkey_transfer.remote(obj=pub_key,
role=consts.HOST,
idx=idx,
suffix=suffix)
host_ciphers[idx] = cipher
return host_ciphers
class FateScript(FederationRuntime):
def __init__(self, fed_obj):
super().__init__(fed_obj.job_id, fed_obj.party_id, fed_obj.role, fed_obj.runtime_conf)
self.trans_conf = file_utils.load_json_conf('contrib/fate_script/conf/FateScriptTransferVar.json')
self.encrypt_operator = None
def remote(self, obj, name: str, tag: str, role=None, idx=-1):
super().remote(obj, name,tag, role, idx)
def get(self, name, tag, idx=-1):
return super().get(name, tag, idx)
def init_encrypt_operator(self):
self.encrypt_operator = PaillierEncrypt()
def init_public_key(self, key_length=1024):
self.encrypt_operator.generate_key(key_length)
return self.encrypt_operator.get_public_key()
def get_public_key(self, public_key):
self.encrypt_operator.set_public_key(public_key)
def __test(self, matrix):
paillierEncrypt = PaillierEncrypt()
paillierEncrypt.generate_key()
publickey = paillierEncrypt.get_public_key()
privatekey = paillierEncrypt.get_privacy_key()
result = distribute_encrypt_matrix(publickey, matrix)
decrypted_result = distribute_decrypt_matrix(privatekey, result)
assert_matrix(matrix, decrypted_result)
def __test_matrix(self, matrix):
paillierEncrypt = PaillierEncrypt()
paillierEncrypt.generate_key()
publickey = paillierEncrypt.get_public_key()
privatekey = paillierEncrypt.get_privacy_key()
enc_matrix = encrypt_matrix(publickey, matrix)
masked_enc_matrix, mask = add_random_mask(enc_matrix)
cleared_enc_matrix = remove_random_mask(masked_enc_matrix, mask)
cleared_matrix = decrypt_matrix(privatekey, cleared_enc_matrix)
assert_matrix(matrix, cleared_matrix)
def __test_scalar(self, value):
paillierEncrypt = PaillierEncrypt()
paillierEncrypt.generate_key()
publickey = paillierEncrypt.get_public_key()
privatekey = paillierEncrypt.get_privacy_key()
enc_value = publickey.encrypt(value)
masked_enc_value, mask = add_random_mask(enc_value)
cleared_enc_value = remove_random_mask(masked_enc_value, mask)
cleared_value = privatekey.decrypt(cleared_enc_value)
print("original matrix", value)
print("cleared_matrix", cleared_value)
self.assertEqual(value, cleared_value)
def setUpClass(cls):
session.init("test_gh_packing")
cls.max_sample_num = 1000
cls.test_num = 10
cls.split_info_test_num = 200
key_length = 1024
sample_id = [i for i in range(cls.max_sample_num)]
# classification data
cls.g, cls.h = generate_bin_gh(cls.max_sample_num)
cls.p_en = PaillierEncrypt()
cls.p_en.generate_key(key_length)
cls.p_packer, cls.p_en_g_l, cls.p_en_h_l, cls.p_en_table, cls.p_collected_gh = \
cls.prepare_testing_data(cls.g, cls.h, cls.p_en, cls.max_sample_num, sample_id, consts.CLASSIFICATION)
cls.compressor = PackedGHCompressor(sync_para=False)
cls.compressor.compressor._padding_length, cls.compressor.compressor._capacity = \
cls.p_packer.packer.cipher_compress_suggest()
print('paillier compress para {}'.format(
cls.p_packer.packer.cipher_compress_suggest()))
# regression data
cls.g_reg, cls.h_reg = generate_reg_gh(cls.max_sample_num, -1000, 1000)
cls.reg_p_packer, cls.reg_p_en_g_l, cls.reg_p_en_h_l, cls.reg_p_en_table, cls.reg_p_collected_gh = \
cls.prepare_testing_data(cls.g_reg, cls.h_reg, cls.p_en, cls.max_sample_num, sample_id, consts.REGRESSION,
g_min=-1000, g_max=1000)
cls.reg_compressor = PackedGHCompressor(sync_para=False)
cls.reg_compressor.compressor._padding_length, cls.reg_compressor.compressor._capacity = \
cls.reg_p_packer.packer.cipher_compress_suggest()
print('paillier compress para {}'.format(
cls.p_packer.packer.cipher_compress_suggest()))
print('initialization done')
def init_encrypt_operator(self):
self.encrypt_operator = PaillierEncrypt()
def gen_paillier_cipher_operator(self, transfer_variables, suffix=tuple()):
self._pubkey_transfer = transfer_variables.paillier_pubkey
pubkey = self._pubkey_transfer.get(idx=0, suffix=suffix)
cipher = PaillierEncrypt()
cipher.set_public_key(pubkey)
return cipher
print("validate_indexes len", len(validate_indexes))
print("test_indexes len", len(test_indexes))
print(
"################################ Build Federated Models ############################"
)
tf.reset_default_graph()
autoencoder_A = Autoencoder(1)
autoencoder_B = Autoencoder(2)
autoencoder_A.build(X_A.shape[-1], 10, learning_rate=0.01)
autoencoder_B.build(X_B.shape[-1], 10, learning_rate=0.01)
paillierEncrypt = PaillierEncrypt()
paillierEncrypt.generate_key()
publickey = paillierEncrypt.get_public_key()
privatekey = paillierEncrypt.get_privacy_key()
# alpha = 100
fake_model_param = FakeFTLModelParam()
partyA = EncryptedFTLGuestModel(autoencoder_A,
fake_model_param,
public_key=publickey)
partyB = EncryptedFTLHostModel(autoencoder_B,
fake_model_param,
public_key=publickey)
federatedLearning = LocalEncryptedFederatedTransferLearning(
partyA, partyB, privatekey)
def generate_encryption_key_pair():
paillierEncrypt = PaillierEncrypt()
paillierEncrypt.generate_key()
public_key = paillierEncrypt.get_public_key()
private_key = paillierEncrypt.get_privacy_key()
return public_key, private_key
def setUp(self):
paillierEncrypt = PaillierEncrypt()
paillierEncrypt.generate_key()
self.publickey = paillierEncrypt.get_public_key()
self.privatekey = paillierEncrypt.get_privacy_key()
def fit(self):
LOGGER.info("@ start arbiter fit")
paillierEncrypt = PaillierEncrypt()
paillierEncrypt.generate_key()
public_key = paillierEncrypt.get_public_key()
private_key = paillierEncrypt.get_privacy_key()
self.private_key = private_key
# distribute public key to guest and host
self._do_remote(public_key,
name=self.transfer_variable.paillier_pubkey.name,
tag=self.transfer_variable.generate_transferid(
self.transfer_variable.paillier_pubkey),
role=consts.HOST,
idx=-1)
self._do_remote(public_key,
name=self.transfer_variable.paillier_pubkey.name,
tag=self.transfer_variable.generate_transferid(
self.transfer_variable.paillier_pubkey),
role=consts.GUEST,
idx=-1)
is_stop = False
start_time = time.time()
while self.n_iter_ < self.max_iter:
# LOGGER.debug("@ iteration: " + str(self.n_iter_))
# decrypt gradient from host
encrypt_host_gradient = self._do_get(
name=self.transfer_variable.encrypt_host_gradient.name,
tag=self.transfer_variable.generate_transferid(
self.transfer_variable.encrypt_host_gradient,
self.n_iter_),
idx=-1)[0]
decrypt_host_gradient = self.__decrypt_gradients(
encrypt_host_gradient)
self._do_remote(
decrypt_host_gradient,
name=self.transfer_variable.decrypt_host_gradient.name,
tag=self.transfer_variable.generate_transferid(
self.transfer_variable.decrypt_host_gradient,
self.n_iter_),
role=consts.HOST,
idx=-1)
# decrypt gradient from guest
encrypt_guest_gradient = self._do_get(
name=self.transfer_variable.encrypt_guest_gradient.name,
tag=self.transfer_variable.generate_transferid(
self.transfer_variable.encrypt_guest_gradient,
self.n_iter_),
idx=-1)[0]
decrypt_guest_gradient = self.__decrypt_gradients(
encrypt_guest_gradient)
self._do_remote(
decrypt_guest_gradient,
name=self.transfer_variable.decrypt_guest_gradient.name,
tag=self.transfer_variable.generate_transferid(
self.transfer_variable.decrypt_guest_gradient,
self.n_iter_),
role=consts.GUEST,
idx=-1)
# decrypt loss from guest
encrypt_loss = self._do_get(
name=self.transfer_variable.encrypt_loss.name,
tag=self.transfer_variable.generate_transferid(
self.transfer_variable.encrypt_loss, self.n_iter_),
idx=-1)[0]
loss = self.__decrypt_loss(encrypt_loss)
if self.converge_func.is_converge(loss):
is_stop = True
# send is_stop indicator to host and guest
self._do_remote(
is_stop,
name=self.transfer_variable.is_encrypted_ftl_stopped.name,
tag=self.transfer_variable.generate_transferid(
self.transfer_variable.is_encrypted_ftl_stopped,
self.n_iter_),
role=consts.HOST,
idx=-1)
self._do_remote(
is_stop,
name=self.transfer_variable.is_encrypted_ftl_stopped.name,
tag=self.transfer_variable.generate_transferid(
self.transfer_variable.is_encrypted_ftl_stopped,
self.n_iter_),
role=consts.GUEST,
idx=-1)
LOGGER.info("@ time: " + str(time.time()) + ", ep: " +
str(self.n_iter_) + ", loss: " + str(loss))
LOGGER.info("@ converged: " + str(is_stop))
self.n_iter_ += 1
if is_stop:
break
end_time = time.time()
LOGGER.info("@ running time: " + str(end_time - start_time))
