def get_compat_shape(type1, type2): """ For tensor types `type1`, `type2` that are of the same rank, return compat_shape (python list) where compat_shape[i] is integer iff type1 and type2 have the same integer shape on dim i. compat_shape[i] is symbolic otherwise. Return None if `type1`, `type2` have different rank or non-tensor type. """ if not types.is_tensor(type1) or not types.is_tensor(type2): return None s1 = type1.get_shape() s2 = type2.get_shape() if len(s1) != len(s2): return None compat_shape = [] for d1, d2 in zip(s1, s2): if d1 != d2: compat_shape.append(get_new_symbol()) else: compat_shape.append(d1) return compat_shape
def types_to_proto(valuetype): if types.is_tensor(valuetype): primitive = types_to_proto_primitive(valuetype.get_primitive()) return create_valuetype_tensor(valuetype.get_shape(), primitive) elif types.is_tuple(valuetype): v_type = pm.ValueType() t_type = v_type.tupleType for t in valuetype.T: new_v_type = t_type.types.add() new_v_type.CopyFrom(types_to_proto(t)) return v_type elif types.is_list(valuetype): elem = valuetype.T[0] length = valuetype.T[1] if types.is_tensor(elem): dtype = types_to_proto_primitive(elem.get_primitive()) elem_shape = elem.get_shape() elif types.is_scalar(elem): dtype = types_to_proto_primitive(valuetype) elem_shape = () elif types.is_str(elem): dtype = types_to_proto_primitive(elem) elem_shape = () else: raise NotImplementedError( "Only list of either tensors or scalars supported. " "Got element of type {}".format(elem.__type_info__())) return create_valuetype_list(length=length, elem_shape=elem_shape, dtype=dtype) elif types.is_dict(valuetype): return create_valuetype_dict(valuetype.T[0], valuetype.T[1]) else: return create_valuetype_scalar(types_to_proto_primitive(valuetype))
def _does_block_contain_symbolic_shape(block): for op in block.operations: for b in op.blocks: if _does_block_contain_symbolic_shape(b): return True for out in op.outputs: if types.is_tensor(out.sym_type): shape = out.sym_type.get_shape() if any_symbolic(shape): return True elif types.is_scalar(out.sym_type) or types.is_str( out.sym_type): if is_symbolic(out.val): return True elif types.is_list(out.sym_type): if types.is_tensor(out.elem_type): if any_symbolic(out.elem_type.get_shape()): return True else: raise NotImplementedError( "\'{}\' type in a list not handled".format( out.elem_type)) else: raise NotImplementedError( "\'{}\' type is not handled".format(out.sym_type)) return False
def promoted_primitive_type(type1, type2): """ Given a pair of tensor or primitive types, find the smallest type that can store an instance of their primitive type. """ ptype1 = type1.get_primitive() if types.is_tensor(type1) else type1 ptype2 = type2.get_primitive() if types.is_tensor(type2) else type2 return types.promote_types(ptype1, ptype2)
def check_output_shapes(x, node): """ x: list[Var] or tuple[Var] node: ParsedTFNode """ if isinstance(x, ListVar): # No check on list. return if not isinstance(x, (list, tuple)): x = [x] tf_shapes = node.attr.get("_output_shapes", None) if tf_shapes is None: return inf_shapes = [] for y in x: if y is None: msg = "TF convert returns None type in TF node {}" raise TypeError(msg.format(node.name)) if types.is_tensor(y.sym_type): inf_shapes.append(list(y.shape)) elif types.is_scalar(y.sym_type): inf_shapes.append([]) else: msg = "Output type {} not understood" raise ValueError(msg.format(y)) for t, s in zip(tf_shapes, inf_shapes): if not compatible_shapes(t, s): msg = ("Op {} ({}) type inference ({}) and TF output shape " + "({}) mismatch") raise ValueError(msg.format(node.name, node.op, s, t))
def get_cast_value(input_var, dtype_val): type_map = { "int32": np.int32, "int64": np.int64, "fp16": np.float16, "fp32": np.float32, "fp64": np.float64, "bool": np.bool, } if dtype_val not in type_map.keys(): raise NotImplementedError( "Parameter dtype of the cast operation can be one of the {}. " "Provided {}".format(type_map.keys(), dtype_val)) if input_var.val is None: if input_var.sym_val is not None and not is_symbolic( input_var.sym_val) and len(input_var.sym_val.shape) == 1: result = [ np.array(val).astype(dtype=type_map[dtype_val]).item() if not is_symbolic(val) else val for val in input_var.sym_val ] return np.array(result) return None if not types.is_tensor(input_var.sym_type): return input_var.val.astype(dtype=type_map[dtype_val]) else: return np.array(input_var.val).astype(dtype=type_map[dtype_val])
def apply(self, prog): user_provided_output_types = prog.main_output_types main_func = prog.functions["main"] output_vars = main_func.outputs if user_provided_output_types is None or len( user_provided_output_types) == 0: return if len(output_vars) != len(user_provided_output_types): msg = "Number of outputs provided by the user, which is {}, " \ "does not match the number of outputs generated by the model, which is {}" raise ValueError( msg.format(len(user_provided_output_types), len(output_vars))) new_outputs = [] for i, output_type in enumerate(user_provided_output_types): required_output_dtype = output_type.dtype output_var = output_vars[i] if required_output_dtype is None or \ not (types.is_tensor(output_var.sym_type) or types.is_scalar(output_var.sym_type)) or \ required_output_dtype == output_var.dtype: # no need to update the output var's dtype in this case new_outputs.append(output_var) else: output_var_name = output_var.name output_var.set_name(output_var_name + "_type_" + types.builtin_to_string(output_var.dtype)) with main_func: output_var = mb.cast( x=output_var, dtype=types.builtin_to_string(required_output_dtype)) output_var.set_name(output_var_name) new_outputs.append(output_var) main_func.set_outputs(new_outputs)
def _adjust_main_outputs(func): new_outputs = [] for output_var in func.outputs: output_type = output_var.sym_type if (types.is_tensor(output_type) or types.is_scalar(output_type)) \ and output_var.dtype != types.fp32 \ and output_var.dtype != types.int32 \ and (func.opset_version < target.iOS16 or output_var.dtype != types.fp16): # since fp16 is a valid output type for coreml from ios16 spec onwards, no need to cast output_dtype_str = types.builtin_to_string(output_var.dtype) supported_dtypes = "{int32, fp32, fp64}" if func.opset_version < target.iOS16 else \ "{int32, fp16, fp32, fp64}" msg = "\nOutput '{}' is of dtype {}. The " +\ "CoreML runtime does not support outputs with this dtype " +\ "(supported dtypes are: {}). This output will be assigned a dtype " +\ "of fp32. A cast will be inserted at the end of the program to convert" +\ "the original output dtype to the dtype supported by the CoreML runtime.\n" if output_var.dtype == types.fp16: msg += "fp16 dtype output is supported if function.opset_version is chosen to be at least " \ "iOS16/macOS13.\n" logging.warning( msg.format( output_var.name, output_dtype_str, supported_dtypes, )) output_var_name = output_var.name output_var.set_name(output_var_name + "__pre__output__fp32__cast") # Convert the output to fp32, and add a cast. output_var = mb.cast(x=output_var, dtype="fp32") output_var.set_name(output_var_name) new_outputs.append(output_var) func.set_outputs(new_outputs)
def _load_function(context, func_spec, spec_version): if not isinstance(func_spec, pm.Function): raise TypeError("Invalid Function spec object") if func_spec.attributes: raise ValueError("Attributes on functions not supported") func_inputs = {} for named_value_type in func_spec.inputs: name = named_value_type.name valuetype = proto_to_types(named_value_type.type) if not types.is_tensor(valuetype): raise ValueError("Functions inputs can only be tensors") func_inputs[name] = Placeholder( sym_shape=valuetype.get_shape(), dtype=valuetype.get_primitive(), name=name ) context.register_var_with_name(name, func_inputs[name].outputs[0]) opset = func_spec.opset if opset not in func_spec.block_specializations: raise ValueError("Missing block specialization for opset {}".format(opset)) with Function(func_inputs, opset_version=_target(spec_version)) as pymil_func: _load_block(context, func_spec.block_specializations[opset]) return pymil_func
def _create_placeholder(node): node.parse_from_attr() shape = [] dtype = node.attr["dtype"] if types.is_tensor(node.datatype): shape = node.datatype.get_shape() shape = tuple(get_new_symbol() if s is None or s < 0 else s for s in shape) return mb.placeholder(shape, dtype=dtype)
def type_str(self): is_tensor = types.is_tensor(self.sym_type) is_list = types.is_list(self.sym_type) if is_tensor: type_string = "(Tensor)" elif is_list: type_string = "(List)" else: type_string = "(Scalar)" return type_string
def type_inference(self): typea = self.x.sym_type typeb = self.y.sym_type primitive_type = promoted_primitive_type(typea, typeb) if primitive_type is None: raise ValueError( "Incompatible primitive types in broadcast operation") primitive_type = self.get_dtype(primitive_type) # broadcast if not types.is_tensor(typea) and not types.is_tensor(typeb): # both typea and typeb are not tensors return primitive_type if types.is_tensor(typea) and not types.is_tensor(typeb): # a is tensor, b is not return types.tensor(primitive_type, typea.get_shape()) if not types.is_tensor(typea) and types.is_tensor(typeb): # a is not tensor, b is return types.tensor(primitive_type, typeb.get_shape()) # both a, b are tensors shapea = list(typea.get_shape()) shapeb = list(typeb.get_shape()) ret_shape = broadcast_shapes(shapea, shapeb) return types.tensor(primitive_type, ret_shape)
def create_immediate_value(var): if types.is_tensor(var.sym_type): return create_tensor_value(var.val) elif types.is_list(var.sym_type): if var.elem_type == types.str: return create_list_scalarvalue(var.val, np.str) elif var.elem_type == types.int64: return create_list_scalarvalue(var.val, np.int64) else: raise NotImplementedError( "List element type, {}, not supported yet.".format( var.sym_type.__type_info__())) else: return create_scalar_value(var.val)
def _adjust_main_inputs(func): first_op = func.operations[0] if len(func.operations) > 0 else None for input_name, input_var in func.inputs.items(): if (types.is_tensor(input_var.sym_type) or types.is_scalar(input_var.sym_type)) \ and input_var.dtype != types.fp32 \ and input_var.dtype != types.int32: input_dtype_str = types.builtin_to_string(input_var.dtype) if types.is_int(input_var.dtype): # Replace non-int32 input type with int32. logging.warning("Input" + input_var.name + " is of dtype " + input_dtype_str +\ ". Only integer variables of bit width 32 are supported by the CoreML runtime. " +\ "This input will be assigned a dtype of int32. " +\ "No cast will be inserted; the previous dtype will be replaced.") _adjust_var_dtype_helper(input_var, types.int32) elif input_var.dtype == types.fp64: # Replace float64 input type with fp32. logging.warning("Input '" + input_var.name + "' is of dtype fp64. 64 bit float inputs are " +\ "not supported by ML program models. This input will be assigned a dtype " +\ "of fp32. No cast will be inserted; the previous dtype will be replaced.") _adjust_var_dtype_helper(input_var, types.fp32) elif input_var.dtype == types.fp16 \ and func.opset_version >= target.iOS16: pass # do nothing, since fp16 is a valid input type for CoreML else: # This is some other dtype. Change the type to fp32 and add a cast. # This is only a limitation of main--other functions do not represent CoreML model inputs # and do not have the same limitation on input types. supported_dtypes = "{int32, fp32, fp64}" if func.opset_version < target.iOS16 else \ "{int32, fp16, fp32, fp64}" msg = "\nInput '{}' is of dtype {}. The " +\ "CoreML runtime does not support inputs with this dtype " +\ "(supported dtypes are: {}). This input will be assigned a dtype of " +\ "fp32. A cast will be inserted at the beginning of the program to " +\ "convert the input to the originally defined dtype.\n" if input_var.dtype == types.fp16: msg += "fp16 dtype input is supported if the function.opset_version is chosen to be at least " \ "iOS16/macOS13.\n" logging.warning( msg.format(input_var.name, input_dtype_str, supported_dtypes)) casted_input_var = mb.cast(x=input_var, dtype=input_dtype_str, before_op=first_op) func.replace_uses_of_var_after_op( anchor_op=casted_input_var.op, old_var=input_var, new_var=casted_input_var) _adjust_var_dtype_helper(input_var, types.fp32)
def value_inference(self): type_map = { "int32": np.int32, "int64": np.int64, "fp32": np.float32, "fp64": np.float64, "bool": np.bool, } if self.dtype.val not in type_map.keys(): raise NotImplementedError( "Parameter dtype of the cast operation can be one of the {}. " "Provided {}".format(type_map.keys(), self.dtype.val)) if not types.is_tensor(self.x.sym_type): return self.x.val.astype(dtype=type_map[self.dtype.val]) else: return np.array(self.x.val).astype(dtype=type_map[self.dtype.val])
def type_inference(self): type_map = { "int32": types.int32, "int64": types.int64, "fp32": types.fp32, "fp64": types.fp64, "bool": types.bool, } if self.dtype.val not in type_map.keys(): raise NotImplementedError( "Parameter dtype of the cast operation can be one of the {}. " "Provided {}".format(type_map.keys(), self.dtype.val)) if not types.is_tensor(self.x.sym_type): return type_map[self.dtype.val] ret_shape = self.x.shape return types.tensor(type_map[self.dtype.val], ret_shape)
def _load_value(context, value_spec): if not isinstance(value_spec, pm.Value): raise TypeError("Invalid Value spec object") if value_spec.docString: raise ValueError("Docstring would get lost in the process.") if value_spec.type.WhichOneof("type") == "tensorType": valuetype = proto_to_types(value_spec.type) is_tensor = types.is_tensor(valuetype) dtype = valuetype if not is_tensor else valuetype.get_primitive() shape = () if not is_tensor else valuetype.get_shape() if value_spec.WhichOneof("value") == "immediateValue": value = _load_immediate_value(value_spec.immediateValue) else: value = _load_file_value(context, value_spec.blobFileValue, dtype) if dtype in (types.fp16, types.int8, types.uint8, types.uint32): value = np.frombuffer(value, types.nptype_from_builtin(dtype)).reshape( shape ) elif dtype == types.str and shape == (): value = str(value[0]) elif dtype in (types.fp32, types.str, types.bool, types.int32, types.int64): value = ( np.array(value).astype(types.nptype_from_builtin(dtype)).reshape(shape) ) else: raise ValueError("Invalid dtype for tensor value") else: raise NotImplementedError("Only value of tensorType implemented yet") if not is_tensor and not isinstance(value, str): value = types.nptype_from_builtin(dtype)(value.item()) return value
def _adjust_main_outputs(func): new_outputs = [] for output_var in func.outputs: output_type = output_var.sym_type if (_types.is_tensor(output_type) or _types.is_scalar(output_type)) \ and output_var.dtype != _types.fp32 \ and output_var.dtype != _types.int32: output_dtype_str = _types.builtin_to_string(output_var.dtype) _warnings.warn("Output" + output_var.name + " is of dType " + output_dtype_str + ". The " +\ "CoreML runtime does not support outputs with this dType (only int32 and " +\ "fp32 are supported for outputs). This output will be assigned a dType " +\ "of fp32. A cast will be inserted at the end of the program to convert" +\ "the original output dType to the dType supported by the CoreML runtime.") output_var_name = output_var.name output_var.set_name(output_var_name + "__pre__output__fp32__cast") # Convert the output to fp32, and add a cast. with func: output_var = _mb.cast(x=output_var, dtype="fp32") output_var.set_name(output_var_name) new_outputs.append(output_var) func.set_outputs(new_outputs)
def _adjust_main_inputs(func): first_op = func.operations[0] if len(func.operations) > 0 else None for input_name, input_var in func.inputs.items(): if (_types.is_tensor(input_var.sym_type) or _types.is_scalar(input_var.sym_type)) \ and input_var.dtype != _types.fp32 \ and input_var.dtype != _types.int32: input_dtype_str = _types.builtin_to_string(input_var.dtype) if _types.is_int(input_var.dtype): # Replace non-int32 input type with int32. _warnings.warn("Input" + input_var.name + " is of dType " + input_dtype_str +\ ". Only integer variables of bit width 32 are supported by the CoreML runtime. " +\ "This input will be assigned a dType of int32. " +\ "No cast will be inserted; the previous dtype will be replaced.") _adjust_var_dtype_helper(input_var, _types.int32) elif input_var.dtype == _types.fp64: # Replace float64 input type with fp32. _warnings.warn("Input" + input_var.name + " is of dtype fp64. 64 bit float inputs are " +\ "not supported by ML program models. This input will be assigned a dType " +\ "of fp32. No cast will be inserted; the previous dtype will be replaced.") _adjust_var_dtype_helper(input_var, _types.fp32) else: # This is some other dType. Change the type to fp32 and add a cast. # This is only a limitation of main--other functions do not represent CoreML model inputs # and do not have the same limitation on input types. _warnings.warn("Input" + input_var.name + " is of dType " + input_dtype_str + ". The " +\ "CoreML runtime does not support inputs with this dType (only fp32 and " +\ "int32 inputs are supported). This input will be assigned a dType of " +\ "fp32. A cast will be inserted at the beginning of the program to " +\ "convert the input to the originally defined dType.") with func: casted_input_var = _mb.cast(x=input_var, dtype=input_dtype_str, before_op=first_op) func.replace_uses_of_var_after_op( anchor_op=casted_input_var.op, old_var=input_var, new_var=casted_input_var) _adjust_var_dtype_helper(input_var, _types.fp32)
def _adjust_var(var): """ Changes the dtype of the provided variable according to the rules outlined in the top level pass comment (see adjust_io_to_supported_types). """ if (types.is_tensor(var.sym_type) or types.is_scalar(var.sym_type)) \ and var.dtype not in __RUNTIME_SUPPORTED_TYPES: dtype_str = types.builtin_to_string(var.dtype) if types.is_int(var.dtype): # Replace non-int32 input type with int32. logging.warning("Input '" + var.name + "' is of dtype " + dtype_str +\ ". Only integer variables of bit width 32 are supported by the CoreML runtime. " +\ "This input will be assigned a dtype of int32. " +\ "No cast will be inserted; the previous dtype will be replaced.") _adjust_var_dtype_helper(var, types.int32) else: # This is some other unsupported dtype. Change the input type to fp32. logging.warning("Var " + var.name + " is of dtype " + dtype_str + ". The CoreML runtime " +\ "does not support this dtype (only fp16, fp32, bool, and int32 are supported). " +\ "This input will be assigned a dtype of fp32. No cast will be inserted; " +\ "the previous dtype will be replaced.") _adjust_var_dtype_helper(var, types.fp32)
def _is_compatible(self, v): return (types.is_list(v.sym_type) or types.is_scalar(v.dtype) or types.is_tensor(v.dtype))
def _load_operation(context, op_spec): if not isinstance(op_spec, pm.Operation): raise TypeError("Invalid Operation spec object") op_type = op_spec.type if op_type == "const" or op_type.startswith("constexpr_"): if op_spec.blocks: raise ValueError("const / constexpr operation can't have any block") if op_spec.inputs: raise ValueError("const / constexpr operation can't have any input") inputs = {k: _load_value(context, v) for k, v in op_spec.attributes.items()} pymil_var = getattr(mb, op_type)(**inputs) context.register_var_with_name(op_spec.outputs[0].name, pymil_var) else: if op_type == "custom_layer": raise NotImplementedError( "Loading Custom Layer operation not yet implemented" ) if op_spec.attributes: raise ValueError("Attributes on operation not supported") # The conversion steps of an operation proto -> PyMIL operation are as following: # (i) Convert the input arguments: # In most of the cases, the input variable is already created beforehand, hence we can # directly access and get them through the TranscriptionContext. # There are cases, though, the inputs are literal value. This could happens in the classify op spec. # For that case, we directly create a constant variable. # (ii) Create nested blocks for control flow operations: # The Python functinoal input arguments for control flow ops cannot be recovered from milproto -> pymil conversion, # for instance, the _body, _cond for mb.while_loop and _true_fn, _false_fn for mb.cond are not invertible # Hence, here we directly create the nested blocks from the proto, and set them to mb.while_loop.blocks / mb.cond.blocks. # Note that, when creating a block, PyMIL required an outer_op, which should be the control flow operation itself. However, # in this approach we take, the outer_op hasn't been created at the time when the blocks produced. Here, we make a "dummy outer_op", # which could pass the check in PyMIL, also it could provide enough information (such as visible variables in the blocks etc.) # for the creation of the block. # (iii) Create PyMIL operation using inputs / blocks # Note that for the control flow cases, we create dummy functional inputs, and use the exisiting block to create the op. # (iv) Set the outer_op for control flow # Once the operation is created, we replace the dummy outer_op with the legit one, to make it a valid PyMIL program inputs = {} for param_name, argument in op_spec.inputs.items(): vars = [] for binding in argument.arguments: binding_type = binding.WhichOneof("binding") if binding_type == "name": vars.append(context.get_var_from_name(binding.name)) elif binding_type == "value": # We only support the list value for now (for the classifier use case) value_spec = binding.value assert value_spec.WhichOneof("value") == "immediateValue" assert value_spec.immediateValue.WhichOneof("value") == "list" list_value = _load_immediate_value(value_spec.immediateValue) values = [] for value_spec in list_value: values.append(_load_value(context, value_spec)) var = mb.const(val=mil_list(values)) vars.append(var) else: raise NotImplementedError("Binding {} not yet implemented".format(binding_type)) op_cls = _SSAOpRegistry._get_core_op_cls(op_type) if len(vars) == 1 and not isinstance( op_cls.input_spec.input_types[param_name], TupleInputType ): inputs[param_name] = vars[0] else: inputs[param_name] = vars blocks = _create_nested_blocks(context, op_spec) _set_inputs_for_control_flow_op(inputs, blocks, op_type) output_var = getattr(mb, op_type)(**inputs) if not isinstance(output_var, (tuple, list)): output_var = [output_var] if len(output_var) != len(op_spec.outputs): raise AssertionError( "Mismatch between number of outputs in operation specification vs PyMIL outputs" ) for spec, var in zip(op_spec.outputs, output_var): context.register_var_with_name(spec.name, var) pymil_type = var.sym_type proto_type = proto_to_types(spec.type) if not types.is_compatible_type(pymil_type, proto_type): # We allow a corner case where the pymil has an 0 rank tensor and the spec produces a scalar if types.is_tensor(pymil_type) and types.is_scalar(proto_type): if pymil_type.get_primitive() == proto_type: continue raise AssertionError( "Mismatch between var types in specification vs PyMIL" ) _set_outer_op_for_nested_blocks(blocks, output_var[0].op)
def shape(self): if types.is_tensor(self._sym_type): return self._sym_type.get_shape() return tuple()
def _is_compatible(self, v): return types.is_tensor(v.sym_type) and v.dtype == types.bool
def dtype(self): if types.is_tensor(self._sym_type): return self._sym_type.get_primitive() return self._sym_type
def _is_compatible(self, v): return types.is_tensor(v.sym_type) and v.dtype in SUPPORT_INT_TYPES
def _is_compatible(self, v): # We only support scalar string type. return types.is_tensor(v.sym_type) and \ v.sym_type.get_primitive() != types.str
def load(prog, weights_dir, resume_on_errors=False, **kwargs): if "main" not in prog.functions: raise ValueError("main function not found in program") mil_passes.mil_backend_passes(prog) # if user has specified "ClassifierConfig", then add the "classify" op to the prog classifier_config = kwargs.get("classifier_config", None) predicted_feature_name = None predicted_probabilities_name = None if classifier_config is not None: predicted_feature_name, predicted_probabilities_name = _add_classify_op( prog, classifier_config) input_types = prog.main_input_types weight_path = os.path.join(weights_dir, _WEIGHTS_FILE_NAME) blob_writer = BlobWriter(weight_path) function_protos = {} for func_name, func in prog.functions.items(): function_protos[func_name] = convert_function(func, prog.parameters, blob_writer) proto = pm.Program( version=1, functions=function_protos, ) input_features = [] output_features = [] symbolic_inputs = [] image_input_names = { } # these are the model inputs marked as image by the user input_shape_map = {} for input_type in input_types: if isinstance(input_type, ImageType): image_input_names[input_type.name] = input_type # error checking for input(s) marked as images if input_type.name not in list( prog.functions["main"].inputs.keys()): msg = "Provided image input '{}' is not one of the inputs of the MIL program" raise ValueError(msg.format(input_type.name)) input_shape_map[input_type.name] = input_type for name, var in prog.functions["main"].inputs.items(): input_feature_type = ft.FeatureType() # error checking for input(s) marked as images # an image input must be of type tensor in program proto # (since an image type does not exist in MIL program) if name in image_input_names and \ not types.is_tensor(var.sym_type): raise ValueError( "For the image input, '{}', its type in the MIL program must be tensor. " "Instead it is {}.".format(name, var.sym_type.__type_info__())) if types.is_tensor(var.sym_type): shape = var.sym_type.get_shape() if any_variadic(shape): raise ValueError( "Variable rank model inputs are not supported!") if any_symbolic(shape): symbolic_inputs.append(name) # We extract the default input shape given by user first if name in input_shape_map: shape = input_shape_map[name].shape.default else: logging.warning( "Input shape not fully specified by enumerated shapes or range dim! 1 will be used for dimension not specified instead." ) # If no input shape is provided (ex. auto conversion of -1 in Tensorflow) shape = [1 if is_symbolic(d) else d for d in shape] if name not in image_input_names: # make a feature type of Type "multiArrayType" array_type = ft.ArrayFeatureType( shape=shape, dataType=cast_to_framework_io_dtype(var, False)) input_feature_type.multiArrayType.CopyFrom(array_type) else: if len(shape) < 3: raise ValueError( "Image input, '{}', must have rank at least 3. Instead it has rank {}" .format(name, len(shape))) # make a feature type of Type "imageType" input_type = image_input_names[name] if not input_type.channel_first: raise ValueError( "Image input, '{}', must be in the channel_first format" .format(name)) if input_type.color_layout == "G": clr_space = ft.ImageFeatureType.ColorSpace.GRAYSCALE elif input_type.color_layout == "BGR": clr_space = ft.ImageFeatureType.ColorSpace.BGR else: clr_space = ft.ImageFeatureType.ColorSpace.RGB image_type = ft.ImageFeatureType(width=shape[-1], height=shape[-2], colorSpace=clr_space) input_feature_type.imageType.CopyFrom(image_type) input_features.append( ml.FeatureDescription(name=name, type=input_feature_type)) elif types.is_scalar(var.sym_type): array_type = ft.ArrayFeatureType( shape=[1], dataType=cast_to_framework_io_dtype(var, False)) input_feature_type.multiArrayType.CopyFrom(array_type) input_features.append( ml.FeatureDescription(name=var.name, type=input_feature_type)) else: raise NotImplementedError() for var in prog.functions["main"].outputs: output_feature_type = ft.FeatureType() if types.is_tensor(var.sym_type) or types.is_primitive(var.sym_type): dataType = None if classifier_config is None or var.name != predicted_feature_name: # Not a classifier output, make sure model output type matches with ML Program type. dataType = cast_to_framework_io_dtype(var, True) else: # Classifier outputs are set up separately, so default to fp32 for now. dataType = ft.ArrayFeatureType.ArrayDataType.FLOAT32 array_type = ft.ArrayFeatureType(shape=None, dataType=dataType) output_feature_type.multiArrayType.CopyFrom(array_type) output_features.append( ml.FeatureDescription(name=var.name, type=output_feature_type)) elif (types.is_dict(var.sym_type)): output_feature_type.dictionaryType.MergeFromString(b"") keytype, valtype = var.sym_type.T if types.is_str(keytype): output_feature_type.dictionaryType.stringKeyType.MergeFromString( b"") elif (keytype == types_int64): output_feature_type.dictionaryType.int64KeyType.MergeFromString( b"") else: raise ValueError("Dictionary key type not supported.") output_features.append( ml.FeatureDescription(name=var.name, type=output_feature_type)) else: raise NotImplementedError() # Model description desc = ml.ModelDescription(input=input_features, output=output_features) if classifier_config is not None: desc.predictedFeatureName = predicted_feature_name desc.predictedProbabilitiesName = predicted_probabilities_name # Manually edit output type of predictedFeatureName. # It doesn't use MLMultiArray and really uses a "primitive" type. for output in desc.output: if output.name == predicted_feature_name: if type(classifier_config.class_labels[0]) == int: output.type.int64Type.MergeFromString(b"") else: output.type.stringType.MergeFromString(b"") break # Create ML Model model = ml.Model(description=desc, specificationVersion=_SPECIFICATION_VERSION_IOS_15) model.mlProgram.CopyFrom(proto) # Set symbolic shapes for input_name in symbolic_inputs: input_type = input_shape_map.get(input_name, None) if isinstance(input_type, ImageType): if isinstance(input_type.shape, EnumeratedShapes): enumerated_shapes = [] for s in input_type.shape.shapes: enumerated_shapes.append( NeuralNetworkImageSize(height=s.shape[-2], width=s.shape[-1])) add_enumerated_image_sizes(model, input_name, sizes=enumerated_shapes) else: img_range = NeuralNetworkImageSizeRange() H = input_type.shape.shape[-2] W = input_type.shape.shape[-1] if isinstance(H, RangeDim): img_range.add_height_range((H.lower_bound, H.upper_bound)) elif is_symbolic(H): img_range.add_height_range((1, -1)) else: img_range.add_height_range((H, H)) if isinstance(W, RangeDim): img_range.add_width_range((W.lower_bound, W.upper_bound)) elif is_symbolic(W): img_range.add_width_range((1, -1)) else: img_range.add_width_range((W, W)) update_image_size_range(model, input_name, img_range) elif isinstance(input_type, TensorType): if isinstance(input_type.shape, EnumeratedShapes): add_multiarray_ndshape_enumeration( model, input_name, [tuple(s.shape) for s in input_type.shape.shapes]) else: lb = [] ub = [] for s in input_type.shape.shape: if isinstance(s, RangeDim): lb.append(s.lower_bound) ub.append(s.upper_bound) elif is_symbolic(s): lb.append(1) ub.append(-1) else: lb.append(s) ub.append(s) set_multiarray_ndshape_range(model, input_name, lower_bounds=lb, upper_bounds=ub) elif input_type is None: sym_type = prog.functions["main"].inputs[input_name].sym_type lb = [] ub = [] for s in sym_type.get_shape(): if is_symbolic(s): lb.append(1) ub.append(-1) else: lb.append(s) ub.append(s) set_multiarray_ndshape_range(model, input_name, lower_bounds=lb, upper_bounds=ub) # Set optional inputs _set_optional_inputs(model, input_types) return model
def _is_compatible(self, v): return types.is_scalar(v.dtype) or types.is_tensor(v.dtype)
def load(prog, **kwargs): if "main" not in prog.functions: msg = "main function not found in program {}" raise ValueError(msg.format(prog)) if len(prog.functions) != 1: msg = ("Program must have exactly one `main` function to " "convert to NN. Program: {}") raise ValueError(msg.format(prog)) nn_backend_passes(prog) input_types = prog.main_input_types output_types = prog.main_output_types v1_inputs = [] symbolic_inputs = {} for name, var in prog.functions["main"].inputs.items(): if types.is_tensor(var.sym_type): sym_shape = var.sym_type.get_shape() if any_variadic(sym_shape): raise NotImplementedError("Variadic rank is not supported") if any_symbolic(sym_shape): user_specified = False for input_type in input_types: if name == input_type.name: sym_shape = input_type.shape.default user_specified = True break # Use dummy static shape, and will set it later. shape = [1 if is_symbolic(d) else d for d in sym_shape] if not user_specified: symbolic_inputs[name] = sym_shape else: shape = sym_shape v1_inputs.append((name, Array(*shape))) elif types.is_scalar(var.sym_type): v1_inputs.append((name, Array(1))) else: raise NotImplementedError() v1_outputs = [] for var in prog.functions["main"].outputs: if types.is_tensor(var.sym_type) or types.is_primitive(var.sym_type): # Disregard the output types v1_outputs.append((var.name, None)) else: raise NotImplementedError() # create neural network builder builder = neural_network.NeuralNetworkBuilder( v1_inputs, v1_outputs, disable_rank5_shape_mapping=True, use_float_arraytype=True, ) # const in V2 are added lazily to V1 by each op whenever needed. # `const_context` stores the const names we've added so far and avoid # adding a const more than once. # const_context: list[set of str] (const name for v1 & v2 # (the same)). Note that in NN in outer layer is visible from the inner # layer, so the const_context is simply a stack of set. const_context = [] # Iterate through ops and add to builder convert_ops( const_context, builder, prog.functions["main"].operations, prog.functions["main"].outputs, ) proto = builder.spec # image input has_image_input = any([isinstance(s, ImageType) for s in input_types]) if has_image_input: proto = _convert_to_image_input(proto, input_types, skip_model_load=kwargs.get( "skip_model_load", False)) # image output if output_types is not None: assert len(output_types) == len(prog.functions["main"].outputs), \ "number of mil program outputs do not match the number of outputs provided by the user" for i, output_proto_desc in enumerate(proto.description.output): output_var = prog.functions["main"].outputs[i] if isinstance(output_types[i], ImageType): if not types.is_tensor(var.sym_type): raise ValueError( "Image output, '{}', is a scalar, but it should be a tensor of rank 4" .format(var.name)) shape = var.sym_type.get_shape() if any_variadic(shape): raise ValueError( "Variable rank model outputs, that are ImageTypes, are not supported" ) if any([is_symbolic(d) for d in shape]): raise NotImplementedError( "Image output '{}' has symbolic dimensions in its shape" .format(var.name)) _validate_image_input_output_shapes( output_types[i].color_layout, shape, var.name, is_input=False) clr_space = _get_colorspace_enum(output_types[i].color_layout) output_proto_desc.type.imageType.colorSpace = clr_space output_proto_desc.type.imageType.width = shape[-1] output_proto_desc.type.imageType.height = shape[-2] # classifier flag classifier_config = kwargs.get("classifier_config", None) if classifier_config is not None: # verify that classifier_config.predicted_probabilities_output if its exists. # And if its empty/None, fill it with the last non const op's output # this is done in "_get_probability_var_for_classifier()" probability_var = _get_probability_var_for_classifier( prog, classifier_config) if classifier_config.predicted_probabilities_output != probability_var.name: classifier_config.predicted_probabilities_output = probability_var.name # add classifier related fields to the proto spec proto = _convert_to_classifier(proto, classifier_config, skip_model_load=kwargs.get( "skip_model_load", False)) _set_user_inputs(proto, input_types) _set_symbolic_inputs(proto, symbolic_inputs) _set_optional_inputs(proto, input_types) return proto