def getAndSetParams(instance: RL, metadata: Dict[str, Any], skipParams: Optional[List[str]] = None) -> None: """ Extract Params from metadata, and set them in the instance. """ # Set user-supplied param values for paramName in metadata["paramMap"]: param = cast("Params", instance).getParam(paramName) if skipParams is None or paramName not in skipParams: paramValue = metadata["paramMap"][paramName] cast("Params", instance).set(param, paramValue) # Set default param values majorAndMinorVersions = VersionUtils.majorMinorVersion( metadata["sparkVersion"]) major = majorAndMinorVersions[0] minor = majorAndMinorVersions[1] # For metadata file prior to Spark 2.4, there is no default section. if major > 2 or (major == 2 and minor >= 4): assert "defaultParamMap" in metadata, ( "Error loading metadata: Expected " + "`defaultParamMap` section not found") for paramName in metadata["defaultParamMap"]: paramValue = metadata["defaultParamMap"][paramName] cast("Params", instance)._setDefault(**{paramName: paramValue})
def getAndSetParams(instance, metadata, skipParams=None): """ Extract Params from metadata, and set them in the instance. """ # Set user-supplied param values for paramName in metadata['paramMap']: param = instance.getParam(paramName) if skipParams is None or paramName not in skipParams: paramValue = metadata['paramMap'][paramName] instance.set(param, paramValue) # Set default param values majorAndMinorVersions = VersionUtils.majorMinorVersion( metadata['sparkVersion']) major = majorAndMinorVersions[0] minor = majorAndMinorVersions[1] # For metadata file prior to Spark 2.4, there is no default section. if major > 2 or (major == 2 and minor >= 4): assert 'defaultParamMap' in metadata, "Error loading metadata: Expected " + \ "`defaultParamMap` section not found" for paramName in metadata['defaultParamMap']: paramValue = metadata['defaultParamMap'][paramName] instance._setDefault(**{paramName: paramValue})
def _cancel_all_jobs(self): if VersionUtils.majorMinorVersion(pyspark.__version__)[0] < 3: # Note: There's bug existing in `sparkContext.cancelJobGroup`. # See https://issues.apache.org/jira/browse/SPARK-31549 warnings.warn("For spark version < 3, pyspark cancelling job API has bugs, " "so we could not terminate running spark jobs correctly. " "See https://issues.apache.org/jira/browse/SPARK-31549 for reference.") else: self._spark.sparkContext.cancelJobGroup(self._job_group)
def run_on_worker_and_fetch_result(): # TODO: handle possible spark exception here. # pylint: disable=fixme rdd = self._spark.sparkContext.parallelize([0], 1) \ .map(lambda _: cloudpickle.dumps(func())) if VersionUtils.majorMinorVersion(pyspark.__version__)[0] < 3: ser_res = rdd.collect()[0] else: ser_res = rdd.collectWithJobGroup(self._job_group, "joblib spark jobs")[0] return cloudpickle.loads(ser_res)
def _get_max_num_concurrent_tasks(self): # maxNumConcurrentTasks() is a package private API # pylint: disable=W0212 pyspark_version = VersionUtils.majorMinorVersion(pyspark.__version__) spark_context = self._spark.sparkContext._jsc.sc() if pyspark_version < (3, 1): return spark_context.maxNumConcurrentTasks() return spark_context.maxNumConcurrentTasks( spark_context.resourceProfileManager().resourceProfileFromId(0))
def getAndSetParams(instance, metadata): """ Extract Params from metadata, and set them in the instance. """ # Set user-supplied param values for paramName in metadata['paramMap']: param = instance.getParam(paramName) paramValue = metadata['paramMap'][paramName] instance.set(param, paramValue) # Set default param values majorAndMinorVersions = VersionUtils.majorMinorVersion(metadata['sparkVersion']) major = majorAndMinorVersions[0] minor = majorAndMinorVersions[1] # For metadata file prior to Spark 2.4, there is no default section. if major > 2 or (major == 2 and minor >= 4): assert 'defaultParamMap' in metadata, "Error loading metadata: Expected " + \ "`defaultParamMap` section not found" for paramName in metadata['defaultParamMap']: paramValue = metadata['defaultParamMap'][paramName] instance._setDefault(**{paramName: paramValue})
def test_parsing_version_string(self): from pyspark.util import VersionUtils self.assertRaises(ValueError, lambda: VersionUtils.majorMinorVersion("abced"))
def test_parsing_version_string(self): from pyspark.util import VersionUtils self.assertRaises(ValueError, lambda: VersionUtils.majorMinorVersion("abced"))
import threading import time import timeit import traceback from hyperopt import base, fmin, Trials from hyperopt.base import validate_timeout, validate_loss_threshold from hyperopt.utils import coarse_utcnow, _get_logger, _get_random_id try: from pyspark.sql import SparkSession from pyspark.util import VersionUtils import pyspark _have_spark = True _spark_major_minor_version = VersionUtils.majorMinorVersion( pyspark.__version__) except ImportError as e: _have_spark = False _spark_major_minor_version = None logger = _get_logger("hyperopt-spark") class SparkTrials(Trials): """ Implementation of hyperopt.Trials supporting distributed execution using Apache Spark clusters. This requires fmin to be run on a Spark cluster. Plugging SparkTrials into hyperopt.fmin() allows hyperopt to send model training and evaluation tasks to Spark workers,