def load_params(self, param):
self.only_output_key = param.only_output_key
self.sync_intersect_ids = param.sync_intersect_ids
self.random_bit = param.random_bit
self.rsa_params = param.rsa_params
self.split_calculation = self.rsa_params.split_calculation
self.random_base_fraction = self.rsa_params.random_base_fraction
self.first_hash_operator = Hash(self.rsa_params.hash_method, False)
self.final_hash_operator = Hash(self.rsa_params.final_hash_method,
False)
self.salt = self.rsa_params.salt
def run_preprocess(self, data_instances):
preprocess_hash_operator = Hash(self.model_param.intersect_preprocess_params.preprocess_method, False)
if self.role == self.model_param.intersect_preprocess_params.filter_owner:
data = self.make_filter_process(data_instances, preprocess_hash_operator)
else:
LOGGER.debug(f"before preprocess, data count: {data_instances.count()}")
data = self.get_filter_process(data_instances, preprocess_hash_operator)
LOGGER.debug(f"after preprocess, data count: {data.count()}")
return data
def load_params(self, param):
# self.only_output_key = param.only_output_key
# self.sync_intersect_ids = param.sync_intersect_ids
super().load_params(param=param)
self.rsa_params = param.rsa_params
self.random_bit = self.rsa_params.random_bit
"""
if param.random_bit is not None and self.random_bit == DEFAULT_RANDOM_BIT:
self.random_bit = param.random_bit
LOGGER.warning(f"param 'random_bit' of IntersectParam will be deprecated in future version, "
f"please use 'random_bit' in RSAParams.")
"""
self.split_calculation = self.rsa_params.split_calculation
self.random_base_fraction = self.rsa_params.random_base_fraction
self.first_hash_operator = Hash(self.rsa_params.hash_method, False)
self.final_hash_operator = Hash(self.rsa_params.final_hash_method,
False)
self.salt = self.rsa_params.salt
def load_params(self, param):
self.only_output_key = param.only_output_key
self.sync_intersect_ids = param.sync_intersect_ids
self.with_encode = param.with_encode
self.encode_params = param.encode_params
self.join_role = param.join_role
self.hash_operator = Hash(param.encode_params.encode_method,
param.encode_params.base64)
self.salt = self.encode_params.salt
def check(self, x):
"""
check whether given instance x exists in bit array
Parameters
----------
x
Returns
-------
"""
hash_encoder = Hash(self.hash_method, False)
for i in range(self.hash_func_count):
ind = int(hash_encoder.compute(x, suffix_salt=self.salt[i]),
16) % self.bit_count
if not self.query_bit(ind):
return False
return True
def load_params(self, param):
# self.only_output_key = param.only_output_key
# self.sync_intersect_ids = param.sync_intersect_ids
super().load_params(param=param)
self.raw_params = param.raw_params
self.use_hash = self.raw_params.use_hash
self.hash_method = self.raw_params.hash_method
self.base64 = self.raw_params.base64
self.salt = self.raw_params.salt
self.join_role = self.raw_params.join_role
self.hash_operator = Hash(self.hash_method, self.base64)
def test_hash(self):
hash_operator = Hash("sha256")
res = str(self.rsa_op2.hash("1", hash_operator))
self.assertEqual(res, "6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b")
def intersect_join_id(self, data_instances):
LOGGER.info("Join id role is {}".format(self.role))
sid_hash_pair = None
if self.with_encode and self.encode_params.encode_method != "none":
if Hash.is_support(self.encode_params.encode_method):
hash_operator = Hash(self.encode_params.encode_method,
self.encode_params.base64)
sid_hash_pair = data_instances.map(
lambda k, v: (hash_operator.compute(
k, postfit_salt=self.encode_params.salt), k))
data_sid = sid_hash_pair.mapValues(lambda v: 1)
else:
raise ValueError(
"Unknown encode_method, please check the configure of hash_param"
)
else:
data_sid = data_instances.mapValues(lambda v: 1)
if self.role == consts.HOST:
send_ids_federation = self.transfer_variable.send_ids_guest
elif self.role == consts.GUEST:
send_ids_federation = self.transfer_variable.send_ids_host
else:
raise ValueError("Unknown intersect role, please check the code")
recv_ids_list = send_ids_federation.get(idx=-1)
ids_list_size = len(recv_ids_list)
LOGGER.info("Get ids_list from role-send, ids_list size is {}".format(
len(recv_ids_list)))
if ids_list_size == 1:
hash_intersect_ids = recv_ids_list[0].join(
data_sid, lambda i, d: "intersect_id")
elif ids_list_size > 1:
hash_intersect_ids_list = []
for ids in recv_ids_list:
hash_intersect_ids_list.append(
ids.join(data_sid, lambda i, d: "intersect_id"))
hash_intersect_ids = self.get_common_intersection(
hash_intersect_ids_list)
else:
hash_intersect_ids = None
LOGGER.info("Finish intersect_ids computing")
if self.sync_intersect_ids:
if self.role == consts.GUEST:
intersect_ids_federation = self.transfer_variable.intersect_ids_guest
send_role = consts.HOST
elif self.role == consts.HOST:
intersect_ids_federation = self.transfer_variable.intersect_ids_host
send_role = consts.GUEST
else:
raise ValueError(
"Unknown intersect role, please check the code")
intersect_ids_federation.remote(hash_intersect_ids,
role=send_role,
idx=-1)
LOGGER.info("Remote intersect ids to role-send")
if self.role == consts.HOST and len(self.host_party_id_list) > 1:
LOGGER.info(
"raw intersect join role is host, and has {} hosts, get the final intersect_ids from guest"
.format(len(self.host_party_id_list)))
hash_intersect_ids = self.transfer_variable.sync_intersect_ids_multi_hosts.get(
idx=0)
if sid_hash_pair:
hash_intersect_ids_map = hash_intersect_ids.join(
sid_hash_pair, lambda r, s: s)
intersect_ids = hash_intersect_ids_map.map(lambda k, v:
(v, 'intersect_id'))
else:
intersect_ids = hash_intersect_ids
if not self.only_output_key:
intersect_ids = self._get_value_from_data(intersect_ids,
data_instances)
if self.task_version_id is not None:
namespace = "#".join([
str(self.guest_party_id),
str(self.host_party_id), "mountain"
])
for k, v in enumerate(recv_ids_list):
table_name = '_'.join([self.task_version_id, str(k)])
self.tracker.job_tracker.save_as_table(v, table_name,
namespace)
LOGGER.info(
"save guest_{}'s id in name:{}, namespace:{}".format(
k, table_name, namespace))
return intersect_ids
def intersect_send_id(self, data_instances):
sid_hash_pair = None
if self.with_encode and self.encode_params.encode_method != "none":
if Hash.is_support(self.encode_params.encode_method):
# hash_operator = Hash(self.encode_params.encode_method, self.encode_params.base64)
sid_hash_pair = data_instances.map(lambda k, v: (
Intersect.hash(k, self.hash_operator, self.salt), k))
data_sid = sid_hash_pair.mapValues(lambda v: 1)
else:
raise ValueError(
"Unknown encode_method, please check the configuration of encode_param"
)
else:
data_sid = data_instances.mapValues(lambda v: 1)
LOGGER.info("Send id role is {}".format(self.role))
if self.role == consts.GUEST:
send_ids_federation = self.transfer_variable.send_ids_guest
recv_role = consts.HOST
elif self.role == consts.HOST:
send_ids_federation = self.transfer_variable.send_ids_host
recv_role = consts.GUEST
else:
raise ValueError("Unknown intersect role, please check the code")
send_ids_federation.remote(data_sid, role=recv_role, idx=-1)
LOGGER.info("Remote data_sid to role-join")
intersect_ids = None
if self.sync_intersect_ids:
if self.role == consts.HOST:
intersect_ids_federation = self.transfer_variable.intersect_ids_guest
elif self.role == consts.GUEST:
intersect_ids_federation = self.transfer_variable.intersect_ids_host
else:
raise ValueError(
"Unknown intersect role, please check the code")
recv_intersect_ids_list = intersect_ids_federation.get(idx=-1)
LOGGER.info("Get intersect ids from role-join!")
ids_list_size = len(recv_intersect_ids_list)
LOGGER.info(
"recv_intersect_ids_list's size is {}".format(ids_list_size))
recv_intersect_ids = self.get_common_intersection(
recv_intersect_ids_list)
if self.role == consts.GUEST and len(self.host_party_id_list) > 1:
LOGGER.info(
"raw intersect send role is guest, and has {} hosts, remote the final intersect_ids to hosts"
.format(len(self.host_party_id_list)))
self.transfer_variable.sync_intersect_ids_multi_hosts.remote(
recv_intersect_ids, role=consts.HOST, idx=-1)
if sid_hash_pair and recv_intersect_ids is not None:
hash_intersect_ids_map = recv_intersect_ids.join(
sid_hash_pair, lambda r, s: s)
intersect_ids = hash_intersect_ids_map.map(lambda k, v:
(v, 'intersect_id'))
else:
intersect_ids = recv_intersect_ids
else:
LOGGER.info("Not Get intersect ids from role-join!")
if not self.only_output_key:
intersect_ids = self._get_value_from_data(intersect_ids,
data_instances)
return intersect_ids
def get_ind_set(self, x):
hash_encoder = Hash(self.hash_method, False)
return set(
int(hash_encoder.compute(x, suffix_salt=self.salt[i]), 16) %
self.bit_count for i in range(self.hash_func_count))
def load_params(self, param):
super().load_params(param=param)
self.dh_params = param.dh_params
self.hash_operator = Hash(param.dh_params.hash_method)
self.salt = self.dh_params.salt
self.key_length = self.dh_params.key_length