def minimize(self,
loss,
global_step=None,
var_list=None,
gate_gradients=1,
aggregation_method=None,
colocate_gradients_with_ops=False,
name=None,
grad_loss=None):
rtt_get_logger().debug('begin to run StaticReplacePass...')
# Create StaticReplacePass
PassObj = StaticReplacePass()
# Run the pass, and return new loss
if isinstance(loss, rtt_ts.RttTensor):
loss = PassObj.run(loss._raw)
else:
loss = PassObj.run(loss)
rtt_get_logger().debug('end to run StaticReplacePass.')
# generate secure gradient graph
train_op = super(tf.train.GradientDescentOptimizer,
self).minimize(loss, global_step, var_list,
gate_gradients, aggregation_method,
colocate_gradients_with_ops, name,
grad_loss)
# generate message id
MsgIdGenerator(regenerate=True).gen_msgid_and_notified(loss)
return train_op
def is_exist_secure_op(self, op):
"""
Does the data flow(subgraph) have one or more Secure node
:param op: the node is one part of the subgraph
:return: True mean exist secure node, otherwise is False
"""
# Take action based on the class of the source instance
if isinstance(op, ops.Tensor):
return self.is_exist_secure_op(op.op)
elif isinstance(op, ops.Operation):
# get the op def name
op_def_name = op.op_def.name
# If the op def name is secure op name, return true
secure_info_val = list(self.secure_ops_infos.values())
for item in secure_info_val:
if op_def_name in item:
return True
# If it has inputs, call this function recursively on each input.
inputs_secure_flag = [self.is_exist_secure_op(x)
for x in op.inputs]
if True in inputs_secure_flag:
return True
else:
return False
else:
# If the op is not Tensor/Operation, return false.
_errmsg = "Unkown the op {} at is_exist_secure_op() function.".format(str(op))
rtt_get_logger().error(_errmsg)
return False
def gen_msgid_and_notified(self, loss):
"""
generate the rosetta message id, and notified message id to player
"""
msg_id = self._generate(loss)
rtt_get_logger().debug(msg_id)
py_msgid_handler = _rtt.msgid_handle.MsgIdHandle()
py_msgid_handler.update_message_id_info(msg_id)
def _create_secure_op_helper(self, src_op, secure_op_name, secure_op_creator_with_attr,
secure_op_creator_without_attr, secure_op_input_num, to_graph):
"""
create secure op helper function
:param src_op: source op instance, it's not secure op
:param secure_op_name: secure op name, don't contain namespace
:param secure_op_creator_with_attr: secure op creator with build-in attribute, eg: SecureMatmul's transpose_a and transpose_b attributes
:param secure_op_creator_without_attr: secure op creator without build-in attribute, eg:SecureMul
:param secure_op_input_num: secure op input numbers
:param to_graph: dest graph
:return: the secure op instance
"""
# The name of the new instance
if self.rtt_scope != '':
new_secure_name = self.rtt_scope + '/' + secure_op_name + str(src_op._id)
new_src_name = self.rtt_scope + '/' + src_op.name
else:
new_secure_name = secure_op_name + str(src_op._id)
new_src_name = src_op.name
# If it has an original_op parameter, copy it
# the original_op must be none.
if src_op._original_op is not None:
raise NotImplementedError("not supported")
# If it has control inputs, call this function recursively on each.
new_control_inputs = [self.copy_and_replace_to_graph(x, to_graph)
for x in src_op.control_inputs]
assert len(new_control_inputs) == 0, "%s don't have control input edges" %secure_op_name
# If it has inputs, call this function recursively on each.
new_inputs = [self.copy_and_replace_to_graph(x, to_graph)
for x in src_op.inputs]
assert len(new_inputs) == secure_op_input_num, "{0} need {1} edges, but real edges is {2}".format(secure_op_name,
secure_op_input_num, len(new_inputs))
# checked secure op inputs
self._checked_secure_op_inputs(secure_op_name, new_inputs)
# create a secure op
try:
if self._is_op_support_const_attr(src_op.op_def.name):
new_op = self._create_secure_op_with_const_attr(src_op, secure_op_creator_with_attr,
secure_op_creator_without_attr, new_inputs,
secure_op_input_num, new_secure_name)
else:
new_op = self._create_secure_op_without_const_attr(src_op, secure_op_creator_with_attr,
secure_op_creator_without_attr, new_inputs,
secure_op_input_num, new_secure_name)
except Exception as e:
rtt_get_logger().error(str(e))
# Save the op info and return the op
self.dc_op_info[new_src_name] = new_op
return new_op
def __get_numpy(self, file: str, is_x: bool, *args, **kwargs):
arr = None
try:
df = pd.read_csv(file, *args, **kwargs)
arr = df.to_numpy()
except FileNotFoundError as e:
pass
rtt_get_logger().error(str(e))
except Exception as e:
pass
# print(e)
return arr
def _deepcopy_op_helper(self, src_op, new_name, new_inputs,
new_ctrl_inputs, new_original_op, to_graph):
"""
deepcopy op helper function
:param src_op: source op instance.
:param new_name: new op name.
:param new_inputs: new op inputs.
:param new_ctrl_inputs: new op control inputs.
:param new_original_op: new original op, Used to associate the new `Operation` with an
# existing `Operation` (for example, a replica with the op that was replicated).
:param to_graph: dest graph
:return: the deep copy op instance
"""
# If the new_name op has already exist, find it and return
if (new_name in self.dc_op_info.keys()):
return self.dc_op_info[new_name]
# Make a new node_def based on that of the original.
# An instance of tensorflow.core.framework.graph_pb2.NodeDef,
# it stores String-based info such as name, device and type of the op.
# Unique to every Operation instance.
new_node_def = deepcopy(src_op.node_def)
# Change the name
new_node_def.name = new_name
# Copy the other inputs needed for initialization
output_types = src_op._output_types[:]
input_types = src_op._input_types[:]
# Make a copy of the op_def too.
# Its unique to every _type_ of Operation.
op_def = deepcopy(src_op.op_def)
# Initialize a new Operation instance
try:
new_op = ops.Operation(new_node_def,
to_graph,
new_inputs,
output_types,
new_ctrl_inputs,
input_types,
new_original_op,
op_def)
except Exception as e:
rtt_get_logger().error(str(e))
# Save the op info
self.dc_op_info[new_name] = new_op
return new_op
def _is_need_secure_op(self, op):
"""
Replace the op with secure op or deep copy the op based on the data flow.
:param op: judge the op is secure op or not based on the data flow
:return: True is secure op , otherwise is False
"""
# The name of the new instance
if self.rtt_scope != '':
new_name = self.rtt_scope + '/' + op.name
else:
new_name = op.name
# If a variable is trainable, so need secure op to replace the op
if new_name in self.all_vars:
if new_name in self.train_vars:
return True
else:
return False
# if the op name is in self.need_secure_op_name,
# so need secure op to replace the op
if new_name in self.need_secure_op_name:
return True
# Take action based on the class of the source instance
if isinstance(op, ops.Tensor):
return self._is_need_secure_op(op.op)
elif isinstance(op, ops.Operation):
# If the op is placeholder, return true
if (op.op_def.name == "Placeholder"):
return True
# If it has inputs, call this function recursively on each.
inputs_secure_flag = [
self._is_need_secure_op(x) for x in op.inputs
]
if True in inputs_secure_flag:
return True
else:
return False
else:
# If the op is not Tensor/Operation, return false.
rtt_get_logger().error("Unkown the op: " + str(op))
return False
def _convert_tensorflow_tensor(tensor):
""" convert tensorflow native tensor to rtt tensor """
if tensor.dtype in (tf.string, ):
# supported as-is
rtt_get_logger().debug("debug in _convert_tensorflow_tensor:" +
str(tensor))
return RttTensor(rtt_ops.tf_to_rtt(tensor))
if tensor.dtype in (tf.int32, tf.int64, tf.float16, tf.float32,
tf.float64):
# supported as strings
tensor = tf.as_string(tensor)
return RttTensor(rtt_ops.tf_to_rtt(tensor))
raise ValueError(
"Don't know how to convert TensorFlow tensor with dtype '{}'".format(
tensor.dtype))
def create_secure_op(self, src_op, to_graph):
"""
Create a secure op based on the source op, e.g.: Mul op => SecureMul op
:param src_op: source op instance.
:param to_graph: dest graph
:return: secure op instance
"""
# get new dest op define name and create functor with the dest secure op define name.
try:
secure_def_name = self.secure_ops_infos[src_op.op_def.name.lower()][self.SECURE_DEF_NAME_IDX]
secure_create_func = self.secure_ops_infos[src_op.op_def.name.lower()][self.SECURE_CREATOR_IDX]
secure_op = self.secure_ops_infos[src_op.op_def.name.lower()][self.SECURE_OP_IDX]
assert secure_create_func != None, "secure creator is none."
return secure_create_func(src_op, secure_def_name, secure_op, to_graph)
except KeyError:
_errmsg = "tf native op {} does not implemented Secure op".format(src_op.op_def.name.lower())
rtt_get_logger().error(_errmsg)
def create_secure_op(self, src_op, to_graph):
"""
Create a secure op based on the source op, e.g.: Mul op => SecureMul op
:param src_op: source op instance.
:param to_graph: dest graph
:return: secure op instance
"""
# get new dest op define name and create functor with the dest secure op define name.
try:
secure_def_name = self.secure_ops_infos[src_op.op_def.name.lower()][self.SECURE_DEF_NAME_IDX]
secure_op_creator_with_attr = self.secure_ops_infos[src_op.op_def.name.lower()][self.SECURE_OP_CREATOR_WITH_ATTR_IDX]
secure_op_creator_without_attr = self.secure_ops_infos[src_op.op_def.name.lower()][self.SECURE_OP_CREATOR_WITHOUT_ATTR_IDX]
secure_op_inputs = self.secure_ops_infos[src_op.op_def.name.lower()][self.SECURE_OP_INPUTS_IDX]
return self._create_secure_op_helper(src_op, secure_def_name, secure_op_creator_with_attr,
secure_op_creator_without_attr, secure_op_inputs, to_graph)
except KeyError:
_errmsg = "tf native op {} does not implemented Secure op".format(src_op.op_def.name.lower())
rtt_get_logger().error(_errmsg)
def save_op(self, filename_tensor, saveables):
rtt_get_logger().debug("DEBUG: save_op")
"""Create an Op to save 'saveables'.
This is intended to be overridden by subclasses that want to generate
different Ops.
Args:
filename_tensor: String Tensor.
saveables: A list of BaseSaverBuilder.SaveableObject objects.
Returns:
An Operation that save the variables.
Raises:
RuntimeError: (implementation detail) if "self._write_version" is an
unexpected value.
"""
# pylint: disable=protected-access
tensor_names = []
tensors = []
tensor_slices = []
for saveable in saveables:
for spec in saveable.specs:
tensor_names.append(spec.name)
tensors.append(spec.tensor)
tensor_slices.append(spec.slice_spec)
if self._write_version == saver_pb2.SaverDef.V1:
return io_ops._save(
filename=filename_tensor,
tensor_names=tensor_names,
tensors=tensors,
tensor_slices=tensor_slices)
elif self._write_version == saver_pb2.SaverDef.V2:
# "filename_tensor" is interpreted *NOT AS A FILENAME*, but as a prefix
# of a V2 checkpoint: e.g. "/fs/train/ckpt-<step>/tmp/worker<i>-<step>".
return SecureSaveV2(filename_tensor, tensor_names, tensor_slices,
tensors)
else:
raise RuntimeError("Unexpected write_version: " + self._write_version)
self.stream.writelines(datas)
self.stream.flush()
def __getattr__(self, attr):
return getattr(self.stream, attr)
sys.stdout = ZeroBufferOut(sys.stdout)
if 'ROSETTA_MPC_128' in os.environ and os.environ['ROSETTA_MPC_128'] == 'ON':
tf_dpass_lib = os.path.dirname(__file__) + '/../lib128/libtf-dpass.so'
else:
tf_dpass_lib = os.path.dirname(__file__) + '/../libtf-dpass.so'
tf_dpass_lib = os.path.dirname(__file__) + '/../libtf-dpass.so'
dpass = None
if 'ROSETTA_DPASS' in os.environ and os.environ['ROSETTA_DPASS'] == 'OFF':
rtt_get_logger().debug('NOT load library: {}, disable dynamic pass.'.format(tf_dpass_lib))
else:
dpass = tf.load_op_library(tf_dpass_lib)
rtt_get_logger().debug('load library: {}'.format(tf_dpass_lib))
__doc__ = """
This is LatticeX Rosetta.
"""
# RTT
from latticex.rosetta.rtt import *
# MPC
from latticex.rosetta.secure import *
# ZK for the future
# HE for the future
def __check(task_id):
if not py_protocol_handler.is_activated(task_id):
errmsg = "Protocol have not activated. See rtt.activate()."
rtt_get_logger().error(errmsg)
raise Exception(errmsg)
def _create_secure_op_helper(self, src_op, secure_op_name,
secure_op_input_num, secure_op_creator,
to_graph):
"""
create secure op helper function
:param src_op: source op instance, it's not secure op
:param secure_op_name: secure op name, don't contain namespace
:param secure_op_input_num: secure op input numbers
:param secure_op_creator: secure op creator
:param to_graph: dest graph
:return: the secure op instance
"""
# The name of the new instance
if self.rtt_scope != '':
new_secure_name = self.rtt_scope + '/' + secure_op_name
new_src_name = self.rtt_scope + '/' + src_op.name
else:
new_secure_name = secure_op_name
new_src_name = src_op.name
# If it has an original_op parameter, copy it
# the original_op must be none.
if src_op._original_op is not None:
raise NotImplementedError("not supported")
# If it has control inputs, call this function recursively on each.
new_control_inputs = [
self.copy_and_replace_to_graph(x, to_graph)
for x in src_op.control_inputs
]
assert len(new_control_inputs
) == 0, "%s don't have control input edges" % secure_op_name
# If it has inputs, call this function recursively on each.
new_inputs = [
self.copy_and_replace_to_graph(x, to_graph) for x in src_op.inputs
]
assert len(
new_inputs
) == secure_op_input_num, "{0} need {1} edges, but real edges is {2}".format(
secure_op_name, secure_op_input_num, len(new_inputs))
# checked secure op inputs
self._checked_secure_op_inputs(secure_op_name, new_inputs)
# create secure op
try:
if secure_op_input_num == 1:
new_op = secure_op_creator(new_inputs[0],
name=new_secure_name).op
elif secure_op_input_num == 2:
if self._is_bin_op_unsupport_const_attr(src_op.op_def.name):
new_op = self._create_unsupport_const_attr_secure_bin_op(
src_op, secure_op_creator, new_inputs, new_secure_name)
else:
new_op = secure_op_creator(
new_inputs[0],
new_inputs[1],
name=new_secure_name,
lh_is_const=self._secure_input_is_const(new_inputs[0]),
rh_is_const=self._secure_input_is_const(
new_inputs[1])).op
elif secure_op_input_num == 3:
new_op = secure_op_creator(new_inputs[0],
new_inputs[1],
new_inputs[2],
name=new_secure_name).op
elif secure_op_input_num == 4:
new_op = secure_op_creator(new_inputs[0],
new_inputs[1],
new_inputs[2],
new_inputs[3],
name=new_secure_name).op
elif secure_op_input_num == 5:
new_op = secure_op_creator(new_inputs[0],
new_inputs[1],
new_inputs[2],
new_inputs[3],
new_inputs[4],
name=new_secure_name).op
elif secure_op_input_num == 6:
new_op = secure_op_creator(new_inputs[0],
new_inputs[1],
new_inputs[2],
new_inputs[3],
new_inputs[4],
new_inputs[5],
name=new_secure_name).op
else:
raise ValueError("the %s op don't have %d inputs" %
secure_op_name % secure_op_input_num)
except Exception as e:
rtt_get_logger().error(str(e))
# Save the op info
self.dc_op_info[new_src_name] = new_op
# Return the op
return new_op
def _is_need_secure_op(self, op):
"""
Replace the op with secure op or deep copy the op based on the data flow.
:param op: judge the op is secure op or not based on the data flow
:return: True is secure op , otherwise is False
"""
# The name of the new instance
if self.rtt_scope != '':
new_name = self.rtt_scope + '/' + op.name
else:
new_name = op.name
# If a variable is trainable, so need secure op to replace the op
if new_name in self.all_vars:
if new_name in self.train_vars:
return True
else:
return False
# if the op name is in self.need_secure_op_name,
# so need secure op to replace the op
if new_name in self.need_secure_op_name:
return True
# Take action based on the class of the source instance
if isinstance(op, ops.Tensor):
return self._is_need_secure_op(op.op)
elif isinstance(op, ops.Operation):
# If the op is placeholder, return true
if (op.op_def.name == "Placeholder"):
return True
# If the op is IteratorV2, subgraph data flow analysis,
# check the subgraph(root op is MakeIterator) has need secure op?
if (op.op_def.name == "IteratorV2"):
for unit_op in tf.get_default_graph().get_operations():
if (unit_op.op_def.name == "MakeIterator"):
assert len(
unit_op.inputs
) == 2, "MakeIterator op inputs is incorrect."
if (unit_op.inputs[0].op == op):
return self._is_need_secure_op(unit_op.inputs[1])
elif (unit_op.inputs[1].op == op):
return self._is_need_secure_op(unit_op.inputs[0])
# If the op is BatchDatasetV2, return true
if (op.op_def.name == "BatchDatasetV2"):
return True
# If it has inputs, call this function recursively on each.
inputs_secure_flag = [
self._is_need_secure_op(x) for x in op.inputs
]
if True in inputs_secure_flag:
return True
else:
return False
else:
# If the op is not Tensor/Operation, return false.
rtt_get_logger().error("Unkown the op: " + str(op))
return False
from latticex.rosetta.controller.controller_base_ import _rtt
from latticex.rosetta.controller.common_util import rtt_get_logger
# A simple way to cope commandline arguments
# ########################### commandline arguments
import argparse
import os
_parser = argparse.ArgumentParser(description="LatticeX Rosetta")
_parser.add_argument('--party_id',
type=int,
help="Party ID",
required=False,
default=-1,
choices=[0, 1, 2])
_parser.add_argument('--cfgfile',
type=str,
help="Config File",
default=os.path.abspath('.') + "/CONFIG.json")
_args, _unparsed = _parser.parse_known_args()
# ###########################
# the party id can also be configed in CONFIG.json file.
# If you config it by commandline, this will override the one in config file.
_party_id = _args.party_id
_cfgfile = _args.cfgfile
rtt_get_logger().info('party id: {} with config json file: {}'.format(
_party_id, _cfgfile))
py_protocol_handler = _rtt.protocol.ProtocolHandler()
