def test_save_n_load_model_parameters(self):
model_parameters = {"Wh": np.array([[1, 2, 3, 4],
[5, 6, 7, 8],
[9, 10, 11, 12]]),
"bh": np.array([1, 1, 1, 1]),
"Wo": np.array([[1, 2, 3, 4],
[5, 6, 7, 8],
[9, 10, 11, 12]]),
"bo": np.array([0, 0, 0, 0]),
"model_meta": {"learning_rate": 0.01,
"input_dim": 100,
"hidden_dim": 64}
}
model_table_name = "table_name_" + str(uuid.uuid1())
model_table_ns = "table_ns_" + str(uuid.uuid1())
save_model_parameters(model_parameters, model_table_name, model_table_ns)
actual_model_parameters = load_model_parameters(model_table_name, model_table_ns)
assert len(model_parameters) == len(actual_model_parameters)
for k in actual_model_parameters.keys():
if k == "model_meta":
print(actual_model_parameters[k])
else:
assert_matrix(actual_model_parameters[k], model_parameters[k])
def test_distributed_calculate_X_plus_Y_1(self):
X = np.array([[1., 2., 3.], [14., 5., 6.], [7., 8., 9.]])
Y = np.array([[1], [-1], [1]])
actual_X_plus_Y = X + Y
X_plus_Y = distribute_compute_X_plus_Y(X, Y)
assert_matrix(actual_X_plus_Y, X_plus_Y)
def test_distributed_calculate_X_plus_Y_2(self):
X = np.array([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]])
Z = np.array([[1., 2., 3.], [1., 2., 3.], [1., 2., 3.]])
actual_X_plus_Z = X + Z
X_plus_Z = distribute_compute_X_plus_Y(X, Z)
assert_matrix(actual_X_plus_Z, X_plus_Z)
def test_local_model_proxy_update_local_model(self):
print("----test_DNNLR_update_local_model----")
X = np.array([[1, 2, 3, 4, 5], [4, 5, 6, 7, 8], [7, 8, 9, 10, 11],
[9, 3, 6, 7, 8]])
index_list = [3, 0, 2, 1]
instance_table = create_instance_table(X, index_list)
Wh = np.array([[2, 4, 6, 8], [3, 5, 7, 9], [8, 9, 12, 14],
[5, 6, 7, 8], [1, 4, 2, 1]])
bh = np.zeros(X.shape[1])
local_model = MockAutoencoder(0)
local_model.build(Wh.shape[0], Wh=Wh, bh=bh)
federation_client = MockFATEFederationClient()
proxy = BaseLocalModelUpdateProxy()
proxy.set_model(local_model)
proxy.set_federation_client(federation_client)
dtable, index_tracking_list = proxy.transform(instance_table)
coef = np.array([6, 8, 10])
gradients = np.array([2, 4, 6, 12])
gradient_table = create_shared_gradient_table(gradients, index_list)
training_info = {
"iteration": 10,
"batch_index": 1,
"index_tracking_list": index_tracking_list,
"is_host": False
}
proxy.update_local_model(gradient_table, instance_table, coef,
**training_info)
gradients = gradients.reshape(len(gradients), 1)
coef = coef.reshape(1, len(coef))
back_grad = np.matmul(gradients, coef)
expected_instances = [None] * 4
expected_back_grad = [None] * 4
for idx, g, x in zip(index_list, back_grad, X):
expected_back_grad[idx] = g
expected_instances[idx] = x
expected_back_grad = np.array(expected_back_grad)
expected_instances = np.array(expected_instances)
actual_back_grad = local_model.get_back_grad()
actual_instances = local_model.get_X()
print("expected_instances:", expected_instances)
print("actual_instances:", actual_instances)
print("expected_back_grad", expected_back_grad)
print("actual_back_grad", actual_back_grad)
assert_matrix(expected_instances, actual_instances)
assert_matrix(expected_back_grad, actual_back_grad)
def __test(self, matrix):
paillierEncrypt = PaillierEncrypt()
paillierEncrypt.generate_key()
publickey = paillierEncrypt.get_public_key()
privatekey = paillierEncrypt.get_privacy_key()
result = distribute_encrypt_matrix(publickey, matrix)
decrypted_result = distribute_decrypt_matrix(privatekey, result)
assert_matrix(matrix, decrypted_result)
def test_distributed_calculate_XY(self):
print("--- test_distributed_calculate_XY ---")
X = np.array([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]])
Y = np.array([[1], [-1], [1]])
actual_XY = X * Y
XY = compute_XY(X, Y)
assert_matrix(actual_XY, XY)
def test_single_autoencoder():
# To run this test, you may first download MINST dataset from kaggle:
# https://www.kaggle.com/ngbolin/mnist-dataset-digit-recognizer
file_path = '../../../../data/MINST/train.csv'
Xtrain, Ytrain, Xtest, Ytest = getKaggleMNIST(file_path)
Xtrain = Xtrain.astype(np.float32)
Xtest = Xtest.astype(np.float32)
_, D = Xtrain.shape
autoencoder = Autoencoder(0)
autoencoder.build(D, 200)
init_op = tf.global_variables_initializer()
with tf.Session() as session:
autoencoder.set_session(session)
session.run(init_op)
autoencoder.fit(Xtrain, epoch=1, show_fig=True)
i = np.random.choice(len(Xtest))
x = Xtest[i]
y = autoencoder.predict([x])
plt.subplot(1, 2, 1)
plt.imshow(x.reshape(28, 28), cmap='gray')
plt.title('Original')
plt.subplot(1, 2, 2)
plt.imshow(y.reshape(28, 28), cmap='gray')
plt.title('Reconstructed')
plt.show()
model_parameters = autoencoder.get_model_parameters()
# test whether autoencoder can be restored from stored model parameters
tf.reset_default_graph()
autoencoder_2 = Autoencoder(0)
autoencoder_2.restore_model(model_parameters)
init_op = tf.global_variables_initializer()
with tf.Session() as session:
autoencoder_2.set_session(session)
session.run(init_op)
y_hat = autoencoder_2.predict([x])
plt.subplot(1, 2, 1)
plt.imshow(y.reshape(28, 28), cmap='gray')
plt.title('Original')
plt.subplot(1, 2, 2)
plt.imshow(y_hat.reshape(28, 28), cmap='gray')
plt.title('Reconstructed')
plt.show()
assert_matrix(y, y_hat)
def test_create_table_with_array_2(self):
feature_count = 10
expect_data = np.random.rand(feature_count)
actual_data = np.zeros((feature_count, 1))
dtable = create_table(expect_data)
for item in dtable.collect():
actual_data[item[0]] = item[1]
assert dtable.count() == feature_count
assert_matrix(np.expand_dims(expect_data, axis=1), actual_data)
def test_create_table_with_array_1(self):
row_count = 10
expect_data = np.random.rand(row_count, 10)
actual_data = np.zeros((row_count, 10))
dtable = create_table(expect_data)
for item in dtable.collect():
actual_data[item[0]] = item[1]
assert dtable.count() == row_count
assert_matrix(expect_data, actual_data)
def test_distributed_compute_XY_plus_Z(self):
print("--- test_distributed_compute_XY_plus_Z ---")
X = np.array([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]])
Y = np.array([[1], [-1], [1]])
Z = np.array([[1., 2., 3.], [1., 2., 3.], [1., 2., 3.]])
actual_XY_plus_Z = X * Y + Z
XY_plus_Z = compute_XY_plus_Z(X, Y, Z)
assert_matrix(actual_XY_plus_Z, XY_plus_Z)
def test_distributed_calculate_XY_2(self):
print("--- test_distributed_calculate_XY_2 ---")
# X has shape (4, 3, 3)
X = np.random.rand(4, 3, 3)
# Y has shape (4, 1, 1)
Y = np.random.rand(4, 1, 1)
actual_XY = X * Y
print(actual_XY, actual_XY.shape)
XY = compute_XY(X, Y)
assert_matrix(actual_XY, XY)
def test_distributed_calculate_XY_1(self):
print("--- test_distributed_calculate_XY_1 ---")
# X has shape (4, 3)
X = np.array([[1., 2., 3.], [4., 5., 6.], [7., 8., 9.], [10, 11, 12]])
# Y has shape (4, 1)
Y = np.array([[2], [1], [-1], [1]])
actual_XY = X * Y
print(actual_XY, actual_XY.shape)
XY = compute_XY(X, Y)
assert_matrix(actual_XY, XY)
def __test_matrix(self, matrix):
paillierEncrypt = PaillierEncrypt()
paillierEncrypt.generate_key()
publickey = paillierEncrypt.get_public_key()
privatekey = paillierEncrypt.get_privacy_key()
enc_matrix = encrypt_matrix(publickey, matrix)
masked_enc_matrix, mask = add_random_mask(enc_matrix)
cleared_enc_matrix = remove_random_mask(masked_enc_matrix, mask)
cleared_matrix = decrypt_matrix(privatekey, cleared_enc_matrix)
assert_matrix(matrix, cleared_matrix)
def test_autoencoder_restore_model(self):
X = np.array([[4, 2, 3], [6, 5, 1], [3, 4, 1], [1, 2, 3]])
_, D = X.shape
tf.reset_default_graph()
autoencoder = Autoencoder(0)
autoencoder.build(D, 5)
init_op = tf.global_variables_initializer()
with tf.Session() as session:
autoencoder.set_session(session)
session.run(init_op)
autoencoder.fit(X, epoch=10)
model_parameters = autoencoder.get_model_parameters()
tf.reset_default_graph()
autoencoder.restore_model(model_parameters)
init_op = tf.global_variables_initializer()
with tf.Session() as session:
autoencoder.set_session(session)
session.run(init_op)
Wh = autoencoder.Wh.eval()
Wo = autoencoder.Wo.eval()
bh = autoencoder.bh.eval()
bo = autoencoder.bo.eval()
assert_matrix(model_parameters["Wh"], Wh)
assert_matrix(model_parameters["Wo"], Wo)
assert_matrix(model_parameters["bh"], bh)
assert_matrix(model_parameters["bo"], bo)
def test_encrypt_matmul_2_dim(self):
X = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float64)
Y = np.array([[10, 11, 12], [13, 14, 15], [16, 17, 18]],
dtype=np.float64)
Z = np.matmul(X, Y)
encrypt_Y = self.encrypt_2d_matrix(Y)
res = distribute_encrypt_matmul_2_ob(X, encrypt_Y)
# decrypt res
decrypt_res = decrypt_matrix(self.privatekey, res)
assert_matrix(Z, decrypt_res)
def test_encrypt_matmul_3_dim_3(self):
X = np.array([[[1, 2, 3]], [[10, 11, 12]]], dtype=np.float64)
Y = np.array([[[10, 11, 12], [13, 14, 15], [16, 17, 18]],
[[19, 20, 21], [22, 23, 24], [25, 26, 27]]],
dtype=np.float64)
Z = np.matmul(X, Y)
encrypt_Y = self.encrypt_3d_matrix(Y)
res = distribute_encrypt_matmul_3(X, encrypt_Y)
decrypt_res = decrypt_matrix(self.privatekey, res)
assert_matrix(Z, decrypt_res)
def __test(host_sample_indexes, guest_sample_indexes, before_overlap_indexes, before_host_nonoverlap_indexes):
host_x_dict = {}
host_label_dict = {}
np.random.seed(100)
for i in host_sample_indexes:
host_x_dict[i] = np.random.rand(1, 3)
host_label_dict[i] = np.random.randint(0, 2)
overlap_samples, nonoverlap_samples = fetch_overlap_data(host_x_dict, before_overlap_indexes,
before_host_nonoverlap_indexes)
overlap_labels, nonoverlap_labels = fetch_overlap_data(host_label_dict, before_overlap_indexes,
before_host_nonoverlap_indexes)
overlap_samples = np.squeeze(overlap_samples)
nonoverlap_samples = np.squeeze(nonoverlap_samples)
overlap_labels = np.expand_dims(overlap_labels, axis=1)
nonoverlap_labels = np.expand_dims(nonoverlap_labels, axis=1)
host_x, overlap_indexes, non_overlap_indexes, host_label = overlapping_samples_converter(host_x_dict,
host_sample_indexes,
guest_sample_indexes,
host_label_dict)
after_conversion_overlap_samples = host_x[overlap_indexes]
after_conversion_nonoverlap_samples = host_x[non_overlap_indexes]
after_conversion_overlap_labels = host_label[overlap_indexes]
after_conversion_nonoverlap_labels = host_label[non_overlap_indexes]
assert_matrix(overlap_samples, after_conversion_overlap_samples)
assert_matrix(nonoverlap_samples, after_conversion_nonoverlap_samples)
assert_matrix(overlap_labels, after_conversion_overlap_labels)
assert_matrix(nonoverlap_labels, after_conversion_nonoverlap_labels)
def test_mask_list_of_values(self):
matrix_1 = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12],
[13, 14, 15], [16, 17, 18], [19, 20, 21]])
matrix_2 = np.array([[[33, 22, 31], [14, 15, 16], [17, 18, 19]],
[[10, 11, 12], [13, 14, 15], [16, 17, 18]]])
matrix_list = [matrix_1, matrix_2]
masked_value_list, mask_list = add_random_mask_for_list_of_values(
matrix_list)
cleared_value_list = remove_random_mask_from_list_of_values(
masked_value_list, mask_list)
for matrix, cleared_matrix in zip(matrix_list, cleared_value_list):
assert_matrix(matrix, cleared_matrix)
def test_local_model_proxy_transform(self):
print("----test_DNNLR_transform----")
# create mock data
X = np.array([[1, 2, 3, 4, 5], [4, 5, 6, 7, 8], [7, 8, 9, 10, 11],
[9, 3, 6, 7, 8]])
index_list = [3, 0, 2, 1]
instance_table = create_instance_table(X, index_list)
# create mock local model
Wh = np.array([[2, 4, 6, 8], [3, 5, 7, 9], [8, 9, 12, 14],
[5, 6, 7, 8], [1, 4, 2, 1]])
bh = np.zeros(X.shape[1])
local_model = MockAutoencoder(0)
local_model.build(Wh.shape[0], Wh=Wh, bh=bh)
# create expected transformed features
trans_features = np.matmul(X, Wh)
# create local model proxy to be tested
proxy = BaseLocalModelUpdateProxy()
proxy.set_model(local_model)
# run function
actual_feature_list = []
trans_feat_dtable, index_tracking_list = proxy.transform(
instance_table)
trans_feat_dict = dict(trans_feat_dtable.collect())
for idx in index_tracking_list:
actual_feature_list.append(trans_feat_dict[idx].features)
actual_trans_features = np.array(actual_feature_list)
expected_features = [None] * 4
for idx, row in zip(index_list, trans_features):
expected_features[idx] = row
expected_trans_features = np.array(expected_features)
# assert results
print("index_tracking_list", index_tracking_list)
print("actual_features", actual_trans_features)
print("expected_trans_features", expected_trans_features)
assert_matrix(expected_trans_features, actual_trans_features)
def test_create_n_guest_generators(self):
X = np.random.rand(600, 33)
y = np.random.rand(600, 1)
overlap_ratio = 0.2
guest_split_ratio = 0.3
guest_feature_num = 16
data_size = X.shape[0]
overlap_size = int(data_size * overlap_ratio)
expected_overlap_indexes = np.array(range(overlap_size))
particular_guest_size = int((data_size - overlap_size) * guest_split_ratio)
expected_guest_size = overlap_size + particular_guest_size
expected_host_size = overlap_size + data_size - expected_guest_size
guest_data_generator, host_data_generator, overlap_indexes = \
create_guest_host_data_generator(X, y,
overlap_ratio=overlap_ratio,
guest_split_ratio=guest_split_ratio,
guest_feature_num=guest_feature_num)
guest_features_dict = {}
guest_labels_dict = {}
guest_instances_indexes = []
guest_count = 0
guest_feature_num = 0
for item in guest_data_generator:
key = item[0]
instance = item[1]
guest_feature_num = instance.features.shape[-1]
guest_count += 1
guest_instances_indexes.append(key)
guest_features_dict[key] = instance.features
guest_labels_dict[key] = instance.label
host_features_dict = {}
host_labels_dict = {}
host_instances_indexes = []
host_count = 0
host_feature_num = 0
for item in host_data_generator:
key = item[0]
instance = item[1]
host_feature_num = instance.features.shape[-1]
host_count += 1
host_instances_indexes.append(key)
host_features_dict[key] = instance.features
host_labels_dict[key] = instance.label
assert_array(expected_overlap_indexes, overlap_indexes)
assert len(expected_overlap_indexes) == len(overlap_indexes)
assert X.shape[-1] == guest_feature_num + host_feature_num
assert expected_guest_size == guest_count
assert expected_host_size == host_count
for index in overlap_indexes:
assert guest_labels_dict[index] == host_labels_dict[index]
assert guest_labels_dict[index] == y[index]
assert_matrix(guest_features_dict[index], X[index, :guest_feature_num].reshape(1, -1))
assert_matrix(host_features_dict[index], X[index, guest_feature_num:].reshape(1, -1))
[0.963102, 1.467675, 0.829202, 0.772457, -0.038076, -0.468613]])
infile = "../../../../examples/data/unittest_data.csv"
ids, X, y = load_data(infile, 0, (2, 8), 1)
ids = np.array(ids, dtype=np.int32)
X = np.array(X, dtype=np.float64)
y = np.array(y, dtype=np.int32)
print("ids shape", ids.shape)
print("X shape", X.shape)
print("y shape", y.shape)
assert_array(expected_ids, ids)
assert_array(expected_y, y)
assert_matrix(expected_X, X)
expected_data = {}
for i, id in enumerate(expected_ids):
expected_data[id] = {"X": expected_X[i], "y": expected_y[i]}
init()
data_table = feed_into_dtable(ids, X, y.reshape(-1, 1), (0, len(ids)),
(0, X.shape[-1]))
for item in data_table.collect():
id = item[0]
inst = item[1]
expected_item = expected_data[id]
X_i = expected_item["X"]
y_i = expected_item["y"]
