def test_non_ascii_variable_name_pipeline(self): data = dedent(""" pclass,survived,name,sex,age,sibsp,parch,ticket,fare,cabin,embarked,boat,body,home.dest 1,1,"A",female,29.0,0,0,24160,211.3375,B5,S,2,,"MO" 1,1,"B",male,0.9167,1,2,113781,151.55,C22 C26,S,11,,"Can" 1,0,"C",female,2.0,1,2,113781,151.55,C22 C26,S,,,"Can" 1,0,"D",male,30.0,1,2,113781,151.55,C22 C26,S,,135.0,"Can" 1,0,"E",female,25.0,1,2,113781,151.55,C22 C26,S,,,"Can" 1,1,"F",male,48.0,0,0,19952,26.55,E12,S,3,,"NY" 1,1,"G",female,63.0,1,0,13502,77.9583,D7,S,10,,"NY" 1,0,"H",male,39.0,0,0,112050,0.0,A36,S,,,"NI" 1,1,"I",female,53.0,2,0,11769,51.4792,C101,S,D,,"NY" 1,0,"J",male,71.0,0,0,PC 17609,49.5042,,C,,22.0,"Uruguay" 1,0,"K",male,47.0,1,0,PC 17757,227.525,C62 C64,C,,124.0,"NY" 1,1,"L",female,18.0,1,0,PC 17757,227.525,C62 C64,C,4,,"NY" 1,1,"M",female,24.0,0,0,PC 17477,69.3,B35,C,9,,"F" 1,1,"N",female,26.0,0,0,19877,78.85,,S,6,, 1,1,"L",male,80.0,0,0,27042,30.0,A23,S,B,,"Yorks" 1,0,"O",male,,0,0,PC 17318,25.925,,S,,,"NY" 1,0,"P",male,24.0,0,1,PC 17558,247.5208,B58 B60,C,,,"PQ" 1,1,"Q",female,50.0,0,1,PC 17558,247.5208,B58 B60,C,6,,"PQ" 1,1,"R",female,32.0,0,0,11813,76.2917,D15,C,8,, 1,0,"S",male,36.0,0,0,13050,75.2417,C6,C,A,,"MN" """).strip(" \n") data = pd.read_csv(StringIO(data)) data.rename(columns={"age": "年齢"}, inplace=True) X = data.drop('survived', axis=1) # y = data['survived'] cols = ['embarked', 'sex', 'pclass', '年齢', 'fare'] X = X[cols] for cat in ['embarked', 'sex', 'pclass']: X[cat].fillna('missing', inplace=True) numeric_features = ['年齢', 'fare'] numeric_transformer = Pipeline( steps=[('imputer', SimpleImputer( strategy='median')), ('scaler', StandardScaler())]) categorical_features = ['embarked', 'sex', 'pclass'] categorical_transformer = Pipeline( steps=[('onehot', OneHotEncoder(handle_unknown='ignore'))]) preprocessor = ColumnTransformer( transformers=[('num', numeric_transformer, numeric_features), ('cat', categorical_transformer, categorical_features)]) preprocessor.fit_transform(X) initial_type = [('pclass', Int64TensorType(shape=[None, 1])), ('sex', StringTensorType(shape=[None, 1])), ('年齢', FloatTensorType(shape=[None, 1])), ('fare', FloatTensorType(shape=[None, 1])), ('embarked', StringTensorType(shape=[None, 1]))] onnx_object = convert_sklearn(preprocessor, initial_types=initial_type, target_opset=TARGET_OPSET) sess = InferenceSession(onnx_object.SerializeToString()) self.assertTrue(sess is not None)
def _guess_type_proto_str(data_type, dims): # This could be moved to onnxconverter_common. if data_type == "tensor(float)": return FloatTensorType(dims) if data_type == "tensor(double)": return DoubleTensorType(dims) if data_type == "tensor(string)": return StringTensorType(dims) if data_type == "tensor(int64)": return Int64TensorType(dims) if data_type == "tensor(int32)": return Int32TensorType(dims) if data_type == "tensor(bool)": return BooleanTensorType(dims) if data_type == "tensor(int8)": return Int8TensorType(dims) if data_type == "tensor(uint8)": return UInt8TensorType(dims) if Complex64TensorType is not None: if data_type == "tensor(complex64)": return Complex64TensorType(dims) if data_type == "tensor(complex128)": return Complex128TensorType(dims) raise NotImplementedError( "Unsupported data_type '{}'. You may raise an issue " "at https://github.com/onnx/sklearn-onnx/issues." "".format(data_type))
def _guess_type_proto(data_type, dims): # This could be moved to onnxconverter_common. for d in dims: if d == 0: raise RuntimeError("Dimension should not be null: {}.".format( list(dims))) if data_type == onnx_proto.TensorProto.FLOAT: return FloatTensorType(dims) if data_type == onnx_proto.TensorProto.DOUBLE: return DoubleTensorType(dims) if data_type == onnx_proto.TensorProto.STRING: return StringTensorType(dims) if data_type == onnx_proto.TensorProto.INT64: return Int64TensorType(dims) if data_type == onnx_proto.TensorProto.INT32: return Int32TensorType(dims) if data_type == onnx_proto.TensorProto.BOOL: return BooleanTensorType(dims) if data_type == onnx_proto.TensorProto.INT8: return Int8TensorType(dims) if data_type == onnx_proto.TensorProto.UINT8: return UInt8TensorType(dims) if Complex64TensorType is not None: if data_type == onnx_proto.TensorProto.COMPLEX64: return Complex64TensorType(dims) if data_type == onnx_proto.TensorProto.COMPLEX128: return Complex128TensorType(dims) raise NotImplementedError( "Unsupported data_type '{}'. You may raise an issue " "at https://github.com/onnx/sklearn-onnx/issues." "".format(data_type))
def _declare_input_variables(topology, raw_model_container, extra_config): # Declare input variables. inputs = [] n_inputs = extra_config[ constants.N_INPUTS] if constants.N_INPUTS in extra_config else 1 if constants.INPUT_NAMES in extra_config: assert n_inputs == len(extra_config[constants.INPUT_NAMES]) if constants.TEST_INPUT in extra_config: from onnxconverter_common.data_types import ( FloatTensorType, DoubleTensorType, Int32TensorType, Int64TensorType, StringTensorType, ) test_input = extra_config[constants.TEST_INPUT] if n_inputs > 1 else [ extra_config[constants.TEST_INPUT] ] for i in range(n_inputs): input = test_input[i] input_name = (extra_config[constants.INPUT_NAMES][i] if constants.INPUT_NAMES in extra_config else "input_{}".format(i)) if input.dtype == np.float32: input_type = FloatTensorType(input.shape) elif input.dtype == np.float64: input_type = DoubleTensorType(input.shape) elif input.dtype == np.int32: input_type = Int32TensorType(input.shape) elif input.dtype == np.int64: input_type = Int64TensorType(input.shape) elif input.dtype.kind in constants.SUPPORTED_STRING_TYPES: input_type = StringTensorType(input.shape) else: raise NotImplementedError( "Type {} not supported. Please fill an issue on https://github.com/microsoft/hummingbird/." .format(input.dtype)) inputs.append( topology.declare_logical_variable(input_name, type=input_type)) else: # We have no information on the input. Sklearn/Spark-ML always gets as input a single dataframe, # therefore by default we start with a single `input` variable input_name = extra_config[constants.INPUT_NAMES][ 0] if constants.TEST_INPUT in extra_config else "input" var = topology.declare_logical_variable(input_name) inputs.append(var) # The object raw_model_container is a part of the topology we're going to return. # We use it to store the inputs of the Sklearn/Spark-ML's computational graph. for variable in inputs: raw_model_container.add_input(variable) return inputs
def _guess_numpy_type(data_type, dims): # This could be moved to onnxconverter_common. if data_type == np.float32: return FloatTensorType(dims) if data_type == np.float64: return DoubleTensorType(dims) if data_type in (np.str_, str, object) or str(data_type) in ('<U1', ) or ( hasattr(data_type, 'type') and data_type.type is np.str_): # noqa return StringTensorType(dims) if data_type in (np.int64, ) or str(data_type) == '<U6': return Int64TensorType(dims) if data_type in (np.int32, ) or str(data_type) in ('<U4', ): # noqa return Int32TensorType(dims) if data_type == np.uint8: return UInt8TensorType(dims) if data_type in (np.bool_, bool): return BooleanTensorType(dims) if data_type in (np.str_, str): return StringTensorType(dims) if data_type == np.int8: return Int8TensorType(dims) if data_type == np.int16: return Int16TensorType(dims) if data_type == np.uint64: return UInt64TensorType(dims) if data_type == np.uint32: return UInt32TensorType(dims) if data_type == np.uint16: return UInt16TensorType(dims) if data_type == np.float16: return Float16TensorType(dims) if Complex64TensorType is not None: if data_type == np.complex64: return Complex64TensorType(dims) if data_type == np.complex128: return Complex128TensorType(dims) raise NotImplementedError( "Unsupported data_type %r (type=%r). You may raise an issue " "at https://github.com/onnx/sklearn-onnx/issues." "" % (data_type, type(data_type)))
def test_onnx_no_test_data_string(self): warnings.filterwarnings("ignore") model = OneHotEncoder() X = np.array([["a", "b", "c"]]) model.fit(X) # Create ONNX-ML model onnx_ml_model = convert_sklearn( model, initial_types=[("input", StringTensorType([X.shape[0], X.shape[1]]))], target_opset=11 ) # Test backends are not case sensitive self.assertRaises(RuntimeError, hummingbird.ml.convert, onnx_ml_model, "onnx")
def _guess_type_proto(data_type, dims): # This could be moved to onnxconverter_common. if data_type == onnx_proto.TensorProto.FLOAT: return FloatTensorType(dims) elif data_type == onnx_proto.TensorProto.DOUBLE: return DoubleTensorType(dims) elif data_type == onnx_proto.TensorProto.STRING: return StringTensorType(dims) elif data_type == onnx_proto.TensorProto.INT64: return Int64TensorType(dims) elif data_type == onnx_proto.TensorProto.INT32: return Int32TensorType(dims) elif data_type == onnx_proto.TensorProto.BOOL: return BooleanTensorType(dims) else: raise NotImplementedError( "Unsupported data_type '{}'. You may raise an issue " "at https://github.com/onnx/sklearn-onnx/issues." "".format(data_type))
def _guess_numpy_type(data_type, dims): # This could be moved to onnxconverter_common. if data_type == np.float32: return FloatTensorType(dims) elif data_type in (np.str, str, object) or str(data_type) in ('<U1', ): # noqa return StringTensorType(dims) elif data_type in (np.int64, np.uint64) or str(data_type) == '<U6': return Int64TensorType(dims) elif data_type in (np.int32, np.uint32) or str(data_type) in ('<U4', ): # noqa return Int32TensorType(dims) elif data_type == np.bool: return BooleanTensorType(dims) else: raise NotImplementedError( "Unsupported data_type '{}'. You may raise an issue " "at https://github.com/onnx/sklearn-onnx/issues." "".format(data_type))
def from_pb(obj): """ Creates a data type from a protobuf object. """ def get_shape(tt): return [ tt.shape.dim[i].dim_value for i in range(len(tt.shape.dim)) ] if hasattr(obj, 'extend'): return [Variable.from_pb(o) for o in obj] name = obj.name if obj.type.tensor_type: tt = obj.type.tensor_type elem = tt.elem_type shape = get_shape(tt) if elem == onnx_proto.TensorProto.FLOAT: ty = FloatTensorType(shape) elif elem == onnx_proto.TensorProto.BOOL: ty = BooleanTensorType(shape) elif elem == onnx_proto.TensorProto.DOUBLE: ty = DoubleTensorType(shape) elif elem == onnx_proto.TensorProto.STRING: ty = StringTensorType(shape) elif elem == onnx_proto.TensorProto.INT64: ty = Int64TensorType(shape) elif elem == onnx_proto.TensorProto.INT32: ty = Int32TensorType(shape) else: raise NotImplementedError("Unsupported type '{}' " "(elem_type={}).".format( type(obj.type.tensor_type), elem)) else: raise NotImplementedError("Unsupported type '{}' as " "a string ({}).".format(type(obj), obj)) return Variable(name, name, None, ty)
from onnxconverter_common.data_types import FloatTensorType, StringTensorType from onnxmltools.convert import convert_xgboost, convert_lightgbm, \ convert_tensorflow env = Env() TRANS_PATH = env.str('TRANS_PATH', './outputs/trans') MODEL_PATH_XGB = env.str('MODEL_PATH_XGB', './outputs/xgb') MODEL_PATH_LGB = env.str('MODEL_PATH_LGB', './outputs/lgb') MODEL_PATH_DCN = env.str('MODEL_PATH_DCN', './outputs/dcn') ONNX_TRANS_PATH = env.str('TRANS_PATH', './outputs/trans.onnx') ONNX_MODEl_PATH_XGB = env.str('ONNX_MODEl_PATH_XGB', './outputs/xgb.onnx') ONNX_MODEl_PATH_LGB = env.str('ONNX_MODEl_PATH_LGB', './outputs/lgb.onnx') ONNX_MODEl_PATH_DCN = env.str('ONNX_MODEl_PATH_DCN', './outputs/dcn.onnx') trans_initial_type = [('num_feat', FloatTensorType([None, 13])), ('cat_feat', StringTensorType([None, 26]))] model_initial_type = [('num_feat', FloatTensorType([None, 39]))] print('convert sklearn transformer') trans = joblib.load(TRANS_PATH) onx = convert_sklearn(trans, initial_types=trans_initial_type) onnx.save(onx, ONNX_TRANS_PATH) print('convert XGBoost model') model = xgb.XGBClassifier() model.load_model(MODEL_PATH_XGB) onx = convert_xgboost(model, initial_types=model_initial_type) onnx.save(onx, ONNX_MODEl_PATH_XGB) print('convert LightGBM model') model = lgb.Booster(model_file=MODEL_PATH_LGB)