def _from_java(cls, java_stage): """ Given a Java TrainValidationSplitModel, create and return a Python wrapper of it. Used for ML persistence. """ # Load information from java_stage to the instance. sc = SparkContext._active_spark_context bestModel = JavaParams._from_java(java_stage.bestModel()) validationMetrics = _java2py(sc, java_stage.validationMetrics()) estimator, epms, evaluator = super(TrainValidationSplitModel, cls)._from_java_impl(java_stage) # Create a new instance of this stage. py_stage = cls( bestModel=bestModel, validationMetrics=validationMetrics)._set(estimator=estimator) py_stage = py_stage._set(estimatorParamMaps=epms)._set( evaluator=evaluator) if java_stage.hasSubModels(): py_stage.subModels = [ JavaParams._from_java(sub_model) for sub_model in java_stage.subModels() ] py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java(cls, java_stage): """ Given a Java TrainValidationSplitModel, create and return a Python wrapper of it. Used for ML persistence. """ # Load information from java_stage to the instance. sc = SparkContext._active_spark_context bestModel = JavaParams._from_java(java_stage.bestModel()) validationMetrics = _java2py(sc, java_stage.validationMetrics()) estimator, epms, evaluator = super(TrainValidationSplitModel, cls)._from_java_impl(java_stage) # Create a new instance of this stage. py_stage = cls(bestModel=bestModel, validationMetrics=validationMetrics) params = { "evaluator": evaluator, "estimator": estimator, "estimatorParamMaps": epms, "trainRatio": java_stage.getTrainRatio(), "seed": java_stage.getSeed(), } for param_name, param_val in params.items(): py_stage = py_stage._set(**{param_name: param_val}) if java_stage.hasSubModels(): py_stage.subModels = [ JavaParams._from_java(sub_model) for sub_model in java_stage.subModels() ] py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java(cls, java_stage): """ Given a Java CrossValidatorModel, create and return a Python wrapper of it. Used for ML persistence. """ sc = SparkContext._active_spark_context bestModel = JavaParams._from_java(java_stage.bestModel()) avgMetrics = _java2py(sc, java_stage.avgMetrics()) estimator, epms, evaluator = super(CrossValidatorModel, cls)._from_java_impl(java_stage) py_stage = cls(bestModel=bestModel, avgMetrics=avgMetrics) params = { "evaluator": evaluator, "estimator": estimator, "estimatorParamMaps": epms, "numFolds": java_stage.getNumFolds(), "foldCol": java_stage.getFoldCol(), "seed": java_stage.getSeed(), } for param_name, param_val in params.items(): py_stage = py_stage._set(**{param_name: param_val}) if java_stage.hasSubModels(): py_stage.subModels = [[ JavaParams._from_java(sub_model) for sub_model in fold_sub_models ] for fold_sub_models in java_stage.subModels()] py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java_impl(cls, java_stage): """ Return Python estimator, estimatorParamMaps, and evaluator from a Java ValidatorParams. """ # Load information from java_stage to the instance. estimator = JavaParams._from_java(java_stage.getEstimator()) evaluator = JavaParams._from_java(java_stage.getEvaluator()) epms = [estimator._transfer_param_map_from_java(epm) for epm in java_stage.getEstimatorParamMaps()] return estimator, epms, evaluator
def meta_estimator_transfer_param_maps_from_java(pyEstimator, javaParamMaps): pyStages = MetaAlgorithmReadWrite.getAllNestedStages(pyEstimator) stagePairs = list(map(lambda stage: (stage, stage._to_java()), pyStages)) sc = SparkContext._active_spark_context pyParamMaps = [] for javaParamMap in javaParamMaps: pyParamMap = dict() for javaPair in javaParamMap.toList(): javaParam = javaPair.param() pyParam = None for pyStage, javaStage in stagePairs: if pyStage._testOwnParam(javaParam.parent(), javaParam.name()): pyParam = pyStage.getParam(javaParam.name()) if pyParam is None: raise ValueError('Resolve param in estimatorParamMaps failed: ' + javaParam.parent() + '.' + javaParam.name()) javaValue = javaPair.value() if sc._jvm.Class.forName("org.apache.spark.ml.util.DefaultParamsWritable") \ .isInstance(javaValue): pyValue = JavaParams._from_java(javaValue) else: pyValue = _java2py(sc, javaValue) pyParamMap[pyParam] = pyValue pyParamMaps.append(pyParamMap) return pyParamMaps
def meta_estimator_transfer_param_maps_from_java(pyEstimator, javaParamMaps): pyStages = MetaAlgorithmReadWrite.getAllNestedStages(pyEstimator) stagePairs = list( map(lambda stage: (stage, stage._to_java()), pyStages)) sc = SparkContext._active_spark_context pyParamMaps = [] for javaParamMap in javaParamMaps: pyParamMap = dict() for javaPair in javaParamMap.toList(): javaParam = javaPair.param() pyParam = None for pyStage, javaStage in stagePairs: if pyStage._testOwnParam(javaParam.parent(), javaParam.name()): pyParam = pyStage.getParam(javaParam.name()) if pyParam is None: raise ValueError( 'Resolve param in estimatorParamMaps failed: ' + javaParam.parent() + '.' + javaParam.name()) javaValue = javaPair.value() if sc._jvm.Class.forName("org.apache.spark.ml.PipelineStage" ).isInstance(javaValue): # Note: JavaParams._from_java support both JavaEstimator/JavaTransformer class # and Estimator/Transformer class which implements `_from_java` static method # (such as OneVsRest, Pipeline class). pyValue = JavaParams._from_java(javaValue) else: pyValue = _java2py(sc, javaValue) pyParamMap[pyParam] = pyValue pyParamMaps.append(pyParamMap) return pyParamMaps
def loadNativeModelFromString(model): """ Load the model from a native LightGBM model string. """ ctx = SparkContext._active_spark_context loader = ctx._jvm.com.microsoft.ml.spark.lightgbm.LightGBMRegressionModel java_model = loader.loadNativeModelFromString(model) return JavaParams._from_java(java_model)
def loadNativeModelFromFile(filename): """ Load the model from a native LightGBM text file. """ ctx = SparkContext._active_spark_context loader = ctx._jvm.com.microsoft.ml.spark.lightgbm.LightGBMClassificationModel java_model = loader.loadNativeModelFromFile(filename) return JavaParams._from_java(java_model)
def loadNativeModelFromFile(filename): """ Load the model from a native LightGBM text file. """ ctx = SparkContext._active_spark_context loader = ctx._jvm.com.microsoft.azure.synapse.ml.lightgbm.LightGBMRankerModel java_model = loader.loadNativeModelFromFile(filename) return JavaParams._from_java(java_model)
def _from_java(cls, java_stage): """ Given a Java CrossValidatorModel, create and return a Python wrapper of it. Used for ML persistence. """ bestModel = JavaParams._from_java(java_stage.bestModel()) estimator, epms, evaluator = super(CrossValidatorModel, cls)._from_java_impl(java_stage) py_stage = cls(bestModel=bestModel).setEstimator(estimator) py_stage = py_stage.setEstimatorParamMaps(epms).setEvaluator(evaluator) if java_stage.hasSubModels(): py_stage.subModels = [[JavaParams._from_java(sub_model) for sub_model in fold_sub_models] for fold_sub_models in java_stage.subModels()] py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java(cls, java_stage): """ Given a Java CrossValidatorModel, create and return a Python wrapper of it. Used for ML persistence. """ bestModel = JavaParams._from_java(java_stage.bestModel()) estimator, epms, evaluator = super(CrossValidatorModel, cls)._from_java_impl(java_stage) py_stage = cls(bestModel=bestModel).setEstimator(estimator) py_stage = py_stage.setEstimatorParamMaps(epms).setEvaluator(evaluator) if java_stage.hasSubModels(): py_stage.subModels = [[JavaParams._from_java(sub_model) for sub_model in fold_sub_models] for fold_sub_models in java_stage.subModels()] py_stage._resetUid(java_stage.uid()) return py_stage
def loadNativeModelFromString(model, labelColName="label", featuresColName="features", predictionColName="prediction"): """ Load the model from a native LightGBM model string. """ ctx = SparkContext._active_spark_context loader = ctx._jvm.com.microsoft.ml.spark.lightgbm.LightGBMRegressionModel java_model = loader.loadNativeModelFromString(model, labelColName, featuresColName, predictionColName) return JavaParams._from_java(java_model)
def _from_java_impl(cls, java_stage): """ Return Python estimator, estimatorParamMaps, and evaluator from a Java ValidatorParams. """ # Load information from java_stage to the instance. estimator = JavaParams._from_java(java_stage.getEstimator()) evaluator = JavaParams._from_java(java_stage.getEvaluator()) if isinstance(estimator, JavaEstimator): epms = [estimator._transfer_param_map_from_java(epm) for epm in java_stage.getEstimatorParamMaps()] elif MetaAlgorithmReadWrite.isMetaEstimator(estimator): # Meta estimator such as Pipeline, OneVsRest epms = _ValidatorSharedReadWrite.meta_estimator_transfer_param_maps_from_java( estimator, java_stage.getEstimatorParamMaps()) else: raise ValueError('Unsupported estimator used in tuning: ' + str(estimator)) return estimator, epms, evaluator
def _from_java(cls, java_stage): """ Given a Java TrainValidationSplitModel, create and return a Python wrapper of it. Used for ML persistence. """ bestModel = JavaParams._from_java(java_stage.getBestModel()) py_stage = cls(bestModel=bestModel) py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java(cls, java_stage): """ Given a Java PipelineModel, create and return a Python wrapper of it. Used for ML persistence. """ # Load information from java_stage to the instance. py_stages = [JavaParams._from_java(s) for s in java_stage.stages()] # Create a new instance of this stage. py_stage = cls(py_stages) py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java(cls, java_stage): """ Given a Java TrainValidationSplitModel, create and return a Python wrapper of it. Used for ML persistence. """ # Load information from java_stage to the instance. bestModel = JavaParams._from_java(java_stage.bestModel()) estimator, epms, evaluator = super(TrainValidationSplitModel, cls)._from_java_impl(java_stage) # Create a new instance of this stage. py_stage = cls(bestModel=bestModel).setEstimator(estimator) py_stage = py_stage.setEstimatorParamMaps(epms).setEvaluator(evaluator) if java_stage.hasSubModels(): py_stage.subModels = [JavaParams._from_java(sub_model) for sub_model in java_stage.subModels()] py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java(cls, java_stage): """ Given a Java TrainValidationSplitModel, create and return a Python wrapper of it. Used for ML persistence. """ # Load information from java_stage to the instance. bestModel = JavaParams._from_java(java_stage.bestModel()) estimator, epms, evaluator = super(TrainValidationSplitModel, cls)._from_java_impl(java_stage) # Create a new instance of this stage. py_stage = cls(bestModel=bestModel).setEstimator(estimator) py_stage = py_stage.setEstimatorParamMaps(epms).setEvaluator(evaluator) if java_stage.hasSubModels(): py_stage.subModels = [JavaParams._from_java(sub_model) for sub_model in java_stage.subModels()] py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java(cls, java_stage): """ Given a Java PipelineModel, create and return a Python wrapper of it. Used for ML persistence. """ # Load information from java_stage to the instance. py_stages = [JavaParams._from_java(s) for s in java_stage.stages()] # Create a new instance of this stage. py_stage = cls(py_stages) py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java(cls, java_stage): """ Given a Java CrossValidatorModel, create and return a Python wrapper of it. Used for ML persistence. """ sc = SparkContext._active_spark_context bestModel = JavaParams._from_java(java_stage.bestModel()) avgMetrics = _java2py(sc, java_stage.avgMetrics()) estimator, epms, evaluator = super(CrossValidatorModel, cls)._from_java_impl(java_stage) py_stage = cls(bestModel=bestModel, avgMetrics=avgMetrics)._set(estimator=estimator) py_stage = py_stage._set(estimatorParamMaps=epms)._set(evaluator=evaluator) if java_stage.hasSubModels(): py_stage.subModels = [[JavaParams._from_java(sub_model) for sub_model in fold_sub_models] for fold_sub_models in java_stage.subModels()] py_stage._resetUid(java_stage.uid()) return py_stage
def loadNativeModelFromFile(filename, labelColName="label", featuresColName="features", predictionColName="prediction"): """ Load the model from a native LightGBM text file. """ ctx = SparkContext._active_spark_context loader = ctx._jvm.com.microsoft.ml.spark.LightGBMRankerModel java_model = loader.loadNativeModelFromFile(filename, labelColName, featuresColName, predictionColName) return JavaParams._from_java(java_model)
def _from_java(cls, java_stage: "JavaObject") -> "Pipeline": """ Given a Java Pipeline, create and return a Python wrapper of it. Used for ML persistence. """ # Create a new instance of this stage. py_stage = cls() # Load information from java_stage to the instance. py_stages: List["PipelineStage"] = [ JavaParams._from_java(s) for s in java_stage.getStages() ] py_stage.setStages(py_stages) py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java(cls, java_stage): """ Given a Java CrossValidatorModel, create and return a Python wrapper of it. Used for ML persistence. """ # Load information from java_stage to the instance. bestModel = JavaParams._from_java(java_stage.bestModel()) estimator, epms, evaluator = super(CrossValidatorModel, cls)._from_java_impl(java_stage) # Create a new instance of this stage. py_stage = cls(bestModel=bestModel)\ .setEstimator(estimator).setEstimatorParamMaps(epms).setEvaluator(evaluator) py_stage._resetUid(java_stage.uid()) return py_stage
def _from_java(cls, java_stage): """ Given a Java CrossValidatorModel, create and return a Python wrapper of it. Used for ML persistence. """ # Load information from java_stage to the instance. bestModel = JavaParams._from_java(java_stage.bestModel()) estimator, epms, evaluator = super(CrossValidatorModel, cls)._from_java_impl(java_stage) # Create a new instance of this stage. py_stage = cls(bestModel=bestModel)\ .setEstimator(estimator).setEstimatorParamMaps(epms).setEvaluator(evaluator) py_stage._resetUid(java_stage.uid()) return py_stage
def _transfer_param_map_from_java(self, javaParamMap): """ Transforms a Java ParamMap into a Python ParamMap. """ sc = SparkContext._active_spark_context paramMap = dict() for pair in javaParamMap.toList(): param = pair.param() if self.hasParam(str(param.name())): java_obj = pair.value() if sc._jvm.Class.forName("org.apache.spark.ml.PipelineStage").isInstance(java_obj): # Note: JavaParams._from_java support both JavaEstimator/JavaTransformer class # and Estimator/Transformer class which implements `_from_java` static method # (such as OneVsRest, Pipeline class). py_obj = JavaParams._from_java(java_obj) else: py_obj = _java2py(sc, java_obj) paramMap[self.getParam(param.name())] = py_obj return paramMap
def getBestModel(self): """ Returns the best model. """ return JavaParams._from_java(self._java_obj.getBestModel())
def getModel(self): """ Get the underlying model. """ return JavaParams._from_java(self._java_obj.getModel())