def compatible_shapes(tf_shape, inf_shape):
def compare_elem(dt, ds):
if dt is None or dt < 0:
return True
elif dt == ds:
return True
else:
return False
if tf_shape is None or any_variadic(inf_shape):
return True
else:
return all(compare_elem(dt, ds) for dt, ds in zip(tf_shape, inf_shape))
def type_inference(self):
x_type = self.x.dtype
perm = self.perm.val
x_shape = np.array(self.x.shape)
if len(perm) != self.x.rank:
msg = "perm should have the same length as rank(x): {} != {}"
raise ValueError(msg.format(len(perm), self.x.rank))
if self.x.rank == 0:
return self.x.sym_type # scalar cannot be transposed
if any_variadic(self.x.shape):
ret_shape = get_new_variadic_symbol()
else:
ret_shape = x_shape[perm]
return types.tensor(x_type, tuple(ret_shape))
def compatible_shapes(tf_shape, inf_shape):
def compare_elem(dt, ds):
if dt is None or dt < 0:
return True
elif dt == ds:
return True
elif is_symbolic(ds):
if is_symbolic(dt) and dt != ds:
logging.warning("Symbolic dim {} and {}".format(ds, dt) +\
" assumed to be equal")
return True
else:
return False
if tf_shape is None or any_variadic(inf_shape):
return True
else:
return all(compare_elem(dt, ds) for dt, ds in zip(tf_shape, inf_shape))
def _remove_symbolic_reshape_block(block):
num_changes = 0
for op in list(block.operations):
for b in op.blocks:
num_changes += _remove_symbolic_reshape_block(b)
if op.op_type != "reshape":
continue
if op.shape.val is not None:
# shape does not contain symbol.
continue
if op.shape.sym_val is None:
# shape is runtime determined.
continue
if len(op.shape.child_ops) > 1:
continue
# Use output shape as `shape`
shape = op.outputs[0].shape
if any_variadic(shape):
msg = ("Cannot reshape to variadic from a compile time " +
"shape argument. Variadic shape can only be achieved " +
"via runtime shape argument. op: {}")
raise ValueError(msg.format(op))
num_symbols = num_symbolic(shape)
if num_symbols > 1:
continue
# Convert the one symbol to -1
integer_shape = [-1 if is_symbolic(i) else i for i in shape]
shape_const = mb.const(
val=integer_shape,
name=op.shape.name + "x",
before_op=op,
)
reshaped = mb.reshape(x=op.x,
shape=shape_const,
name=op.name,
before_op=op)
op.enclosing_block.replace_uses_of_var_after_op(anchor_op=op,
old_var=op.outputs[0],
new_var=reshaped)
# Remove all the ops at once
block.remove_ops([op, op.shape.op])
num_changes += 1
return num_changes
def load(prog, **kwargs):
if "main" not in prog.functions:
msg = "main function not found in program {}"
raise ValueError(msg.format(prog))
if len(prog.functions) != 1:
msg = ("Program must have exactly one `main` function to "
"convert to NN. Program: {}")
raise ValueError(msg.format(prog))
nn_backend_passes(prog)
input_types = prog.main_input_types
output_types = prog.main_output_types
v1_inputs = []
symbolic_inputs = {}
for name, var in prog.functions["main"].inputs.items():
if types.is_tensor(var.sym_type):
sym_shape = var.sym_type.get_shape()
if any_variadic(sym_shape):
raise NotImplementedError("Variadic rank is not supported")
if any_symbolic(sym_shape):
user_specified = False
for input_type in input_types:
if name == input_type.name:
sym_shape = input_type.shape.default
user_specified = True
break
# Use dummy static shape, and will set it later.
shape = [1 if is_symbolic(d) else d for d in sym_shape]
if not user_specified:
symbolic_inputs[name] = sym_shape
else:
shape = sym_shape
v1_inputs.append((name, Array(*shape)))
elif types.is_scalar(var.sym_type):
v1_inputs.append((name, Array(1)))
else:
raise NotImplementedError()
v1_outputs = []
for var in prog.functions["main"].outputs:
if types.is_tensor(var.sym_type) or types.is_primitive(var.sym_type):
# Disregard the output types
v1_outputs.append((var.name, None))
else:
raise NotImplementedError()
# create neural network builder
builder = neural_network.NeuralNetworkBuilder(
v1_inputs,
v1_outputs,
disable_rank5_shape_mapping=True,
use_float_arraytype=True,
)
# const in V2 are added lazily to V1 by each op whenever needed.
# `const_context` stores the const names we've added so far and avoid
# adding a const more than once.
# const_context: list[set of str] (const name for v1 & v2
# (the same)). Note that in NN in outer layer is visible from the inner
# layer, so the const_context is simply a stack of set.
const_context = []
# Iterate through ops and add to builder
convert_ops(
const_context,
builder,
prog.functions["main"].operations,
prog.functions["main"].outputs,
)
proto = builder.spec
# image input
has_image_input = any([isinstance(s, ImageType) for s in input_types])
if has_image_input:
proto = _convert_to_image_input(proto,
input_types,
skip_model_load=kwargs.get(
"skip_model_load", False))
# image output
if output_types is not None:
assert len(output_types) == len(prog.functions["main"].outputs), \
"number of mil program outputs do not match the number of outputs provided by the user"
for i, output_proto_desc in enumerate(proto.description.output):
output_var = prog.functions["main"].outputs[i]
if isinstance(output_types[i], ImageType):
if not types.is_tensor(var.sym_type):
raise ValueError(
"Image output, '{}', is a scalar, but it should be a tensor of rank 4"
.format(var.name))
shape = var.sym_type.get_shape()
if any_variadic(shape):
raise ValueError(
"Variable rank model outputs, that are ImageTypes, are not supported"
)
if any([is_symbolic(d) for d in shape]):
raise NotImplementedError(
"Image output '{}' has symbolic dimensions in its shape"
.format(var.name))
_validate_image_input_output_shapes(
output_types[i].color_layout,
shape,
var.name,
is_input=False)
clr_space = _get_colorspace_enum(output_types[i].color_layout)
output_proto_desc.type.imageType.colorSpace = clr_space
output_proto_desc.type.imageType.width = shape[-1]
output_proto_desc.type.imageType.height = shape[-2]
# classifier flag
classifier_config = kwargs.get("classifier_config", None)
if classifier_config is not None:
# verify that classifier_config.predicted_probabilities_output if its exists.
# And if its empty/None, fill it with the last non const op's output
# this is done in "_get_probability_var_for_classifier()"
probability_var = _get_probability_var_for_classifier(
prog, classifier_config)
if classifier_config.predicted_probabilities_output != probability_var.name:
classifier_config.predicted_probabilities_output = probability_var.name
# add classifier related fields to the proto spec
proto = _convert_to_classifier(proto,
classifier_config,
skip_model_load=kwargs.get(
"skip_model_load", False))
_set_user_inputs(proto, input_types)
_set_symbolic_inputs(proto, symbolic_inputs)
_set_optional_inputs(proto, input_types)
return proto
def load(prog, weights_dir, resume_on_errors=False, **kwargs):
if "main" not in prog.functions:
raise ValueError("main function not found in program")
mil_passes.mil_backend_passes(prog)
# if user has specified "ClassifierConfig", then add the "classify" op to the prog
classifier_config = kwargs.get("classifier_config", None)
predicted_feature_name = None
predicted_probabilities_name = None
if classifier_config is not None:
predicted_feature_name, predicted_probabilities_name = _add_classify_op(
prog, classifier_config)
input_types = prog.main_input_types
weight_path = os.path.join(weights_dir, _WEIGHTS_FILE_NAME)
blob_writer = BlobWriter(weight_path)
function_protos = {}
for func_name, func in prog.functions.items():
function_protos[func_name] = convert_function(func, prog.parameters,
blob_writer)
proto = pm.Program(
version=1,
functions=function_protos,
)
input_features = []
output_features = []
symbolic_inputs = []
image_input_names = {
} # these are the model inputs marked as image by the user
input_shape_map = {}
for input_type in input_types:
if isinstance(input_type, ImageType):
image_input_names[input_type.name] = input_type
# error checking for input(s) marked as images
if input_type.name not in list(
prog.functions["main"].inputs.keys()):
msg = "Provided image input '{}' is not one of the inputs of the MIL program"
raise ValueError(msg.format(input_type.name))
input_shape_map[input_type.name] = input_type
for name, var in prog.functions["main"].inputs.items():
input_feature_type = ft.FeatureType()
# error checking for input(s) marked as images
# an image input must be of type tensor in program proto
# (since an image type does not exist in MIL program)
if name in image_input_names and \
not types.is_tensor(var.sym_type):
raise ValueError(
"For the image input, '{}', its type in the MIL program must be tensor. "
"Instead it is {}.".format(name, var.sym_type.__type_info__()))
if types.is_tensor(var.sym_type):
shape = var.sym_type.get_shape()
if any_variadic(shape):
raise ValueError(
"Variable rank model inputs are not supported!")
if any_symbolic(shape):
symbolic_inputs.append(name)
# We extract the default input shape given by user first
if name in input_shape_map:
shape = input_shape_map[name].shape.default
else:
logging.warning(
"Input shape not fully specified by enumerated shapes or range dim! 1 will be used for dimension not specified instead."
)
# If no input shape is provided (ex. auto conversion of -1 in Tensorflow)
shape = [1 if is_symbolic(d) else d for d in shape]
if name not in image_input_names:
# make a feature type of Type "multiArrayType"
array_type = ft.ArrayFeatureType(
shape=shape,
dataType=cast_to_framework_io_dtype(var, False))
input_feature_type.multiArrayType.CopyFrom(array_type)
else:
if len(shape) < 3:
raise ValueError(
"Image input, '{}', must have rank at least 3. Instead it has rank {}"
.format(name, len(shape)))
# make a feature type of Type "imageType"
input_type = image_input_names[name]
if not input_type.channel_first:
raise ValueError(
"Image input, '{}', must be in the channel_first format"
.format(name))
if input_type.color_layout == "G":
clr_space = ft.ImageFeatureType.ColorSpace.GRAYSCALE
elif input_type.color_layout == "BGR":
clr_space = ft.ImageFeatureType.ColorSpace.BGR
else:
clr_space = ft.ImageFeatureType.ColorSpace.RGB
image_type = ft.ImageFeatureType(width=shape[-1],
height=shape[-2],
colorSpace=clr_space)
input_feature_type.imageType.CopyFrom(image_type)
input_features.append(
ml.FeatureDescription(name=name, type=input_feature_type))
elif types.is_scalar(var.sym_type):
array_type = ft.ArrayFeatureType(
shape=[1], dataType=cast_to_framework_io_dtype(var, False))
input_feature_type.multiArrayType.CopyFrom(array_type)
input_features.append(
ml.FeatureDescription(name=var.name, type=input_feature_type))
else:
raise NotImplementedError()
for var in prog.functions["main"].outputs:
output_feature_type = ft.FeatureType()
if types.is_tensor(var.sym_type) or types.is_primitive(var.sym_type):
dataType = None
if classifier_config is None or var.name != predicted_feature_name:
# Not a classifier output, make sure model output type matches with ML Program type.
dataType = cast_to_framework_io_dtype(var, True)
else:
# Classifier outputs are set up separately, so default to fp32 for now.
dataType = ft.ArrayFeatureType.ArrayDataType.FLOAT32
array_type = ft.ArrayFeatureType(shape=None, dataType=dataType)
output_feature_type.multiArrayType.CopyFrom(array_type)
output_features.append(
ml.FeatureDescription(name=var.name, type=output_feature_type))
elif (types.is_dict(var.sym_type)):
output_feature_type.dictionaryType.MergeFromString(b"")
keytype, valtype = var.sym_type.T
if types.is_str(keytype):
output_feature_type.dictionaryType.stringKeyType.MergeFromString(
b"")
elif (keytype == types_int64):
output_feature_type.dictionaryType.int64KeyType.MergeFromString(
b"")
else:
raise ValueError("Dictionary key type not supported.")
output_features.append(
ml.FeatureDescription(name=var.name, type=output_feature_type))
else:
raise NotImplementedError()
# Model description
desc = ml.ModelDescription(input=input_features, output=output_features)
if classifier_config is not None:
desc.predictedFeatureName = predicted_feature_name
desc.predictedProbabilitiesName = predicted_probabilities_name
# Manually edit output type of predictedFeatureName.
# It doesn't use MLMultiArray and really uses a "primitive" type.
for output in desc.output:
if output.name == predicted_feature_name:
if type(classifier_config.class_labels[0]) == int:
output.type.int64Type.MergeFromString(b"")
else:
output.type.stringType.MergeFromString(b"")
break
# Create ML Model
model = ml.Model(description=desc,
specificationVersion=_SPECIFICATION_VERSION_IOS_15)
model.mlProgram.CopyFrom(proto)
# Set symbolic shapes
for input_name in symbolic_inputs:
input_type = input_shape_map.get(input_name, None)
if isinstance(input_type, ImageType):
if isinstance(input_type.shape, EnumeratedShapes):
enumerated_shapes = []
for s in input_type.shape.shapes:
enumerated_shapes.append(
NeuralNetworkImageSize(height=s.shape[-2],
width=s.shape[-1]))
add_enumerated_image_sizes(model,
input_name,
sizes=enumerated_shapes)
else:
img_range = NeuralNetworkImageSizeRange()
H = input_type.shape.shape[-2]
W = input_type.shape.shape[-1]
if isinstance(H, RangeDim):
img_range.add_height_range((H.lower_bound, H.upper_bound))
elif is_symbolic(H):
img_range.add_height_range((1, -1))
else:
img_range.add_height_range((H, H))
if isinstance(W, RangeDim):
img_range.add_width_range((W.lower_bound, W.upper_bound))
elif is_symbolic(W):
img_range.add_width_range((1, -1))
else:
img_range.add_width_range((W, W))
update_image_size_range(model, input_name, img_range)
elif isinstance(input_type, TensorType):
if isinstance(input_type.shape, EnumeratedShapes):
add_multiarray_ndshape_enumeration(
model, input_name,
[tuple(s.shape) for s in input_type.shape.shapes])
else:
lb = []
ub = []
for s in input_type.shape.shape:
if isinstance(s, RangeDim):
lb.append(s.lower_bound)
ub.append(s.upper_bound)
elif is_symbolic(s):
lb.append(1)
ub.append(-1)
else:
lb.append(s)
ub.append(s)
set_multiarray_ndshape_range(model,
input_name,
lower_bounds=lb,
upper_bounds=ub)
elif input_type is None:
sym_type = prog.functions["main"].inputs[input_name].sym_type
lb = []
ub = []
for s in sym_type.get_shape():
if is_symbolic(s):
lb.append(1)
ub.append(-1)
else:
lb.append(s)
ub.append(s)
set_multiarray_ndshape_range(model,
input_name,
lower_bounds=lb,
upper_bounds=ub)
# Set optional inputs
_set_optional_inputs(model, input_types)
return model
def load(prog, **kwargs):
if "main" not in prog.functions:
msg = "main function not found in program {}"
raise ValueError(msg.format(prog))
if len(prog.functions) != 1:
msg = ("Program must have exactly one `main` function to "
"convert to NN. Program: {}")
raise ValueError(msg.format(prog))
nn_backend_passes(prog)
input_types = prog.main_input_types
v1_inputs = []
symbolic_inputs = {}
for name, var in prog.functions["main"].inputs.items():
if types.is_tensor(var.sym_type):
sym_shape = var.sym_type.get_shape()
if any_variadic(sym_shape):
# TODO: rdar://59559656
raise NotImplementedError("Variadic rank is not supported")
if any_symbolic(sym_shape):
user_specified = False
for input_type in input_types:
if name == input_type.name:
sym_shape = input_type.shape.default
user_specified = True
break
# Use dummy static shape, and will set it later.
shape = [1 if is_symbolic(d) else d for d in sym_shape]
if not user_specified:
symbolic_inputs[name] = sym_shape
else:
shape = sym_shape
v1_inputs.append((name, datatypes.Array(*shape)))
elif types.is_scalar(var.sym_type):
v1_inputs.append((name, datatypes.Array(1)))
else:
raise NotImplementedError()
v1_outputs = []
for var in prog.functions["main"].outputs:
if types.is_tensor(var.sym_type) or types.is_primitive(var.sym_type):
# Disregard the output types
v1_outputs.append((var.name, None))
else:
raise NotImplementedError()
# create neural network builder
builder = neural_network.NeuralNetworkBuilder(
v1_inputs,
v1_outputs,
disable_rank5_shape_mapping=True,
use_float_arraytype=True,
)
# const in V2 are added lazily to V1 by each op whenever needed.
# `const_context` stores the const names we've added so far and avoid
# adding a const more than once.
# const_context: list[set of str] (const name for v1 & v2
# (the same)). Note that in NN in outer layer is visible from the inner
# layer, so the const_context is simply a stack of set.
const_context = []
# Iterate through ops and add to builder
convert_ops(
const_context,
builder,
prog.functions["main"].operations,
prog.functions["main"].outputs,
)
# Replace model outputs's name with v1_outputs
output_names = [x[0] for x in v1_outputs]
for i, spec_layer in enumerate(builder.nn_spec.layers):
for j, name in enumerate(spec_layer.output):
for output_name in output_names:
if output_name.split(":")[0] == name:
spec_layer.output[j] = output_name
proto = builder.spec
# image input
has_image_input = any([isinstance(s, ImageType) for s in input_types])
if has_image_input:
proto = _convert_to_image_input(proto, input_types)
# classifier flag
classifier_config = kwargs.get("classifier_config", None)
if classifier_config is not None:
proto = _convert_to_classifier(proto, classifier_config)
_set_user_inputs(proto, input_types)
_set_symbolic_inputs(proto, symbolic_inputs)
return proto
