def getVersion():
here = path.abspath(path.dirname(__file__))
if '.zip' in here:
with zipfile.ZipFile(here[:-len("ai/h2o/sparkling/")],
'r') as archive:
version = archive.read('ai/h2o/sparkling/version.txt').decode(
'utf-8').strip()
else:
with open(path.join(here, 'version.txt'), encoding='utf-8') as f:
version = f.read().strip()
pySparklingVersionComponents = VersionComponents.parseFromSparklingWaterVersion(
version)
pySparkVersionComponents = VersionComponents.parseFromPySparkVersion(
pyspark.__version__)
sparkVersionFromPySparkling = pySparklingVersionComponents.sparkMajorMinorVersion
sparkVersionFromPySpark = pySparkVersionComponents.sparkMajorMinorVersion
if not (sparkVersionFromPySpark == sparkVersionFromPySparkling):
raise Exception("""
You are using PySparkling for Spark {}, but your PySpark is of version {}.
Please make sure Spark and PySparkling versions are compatible.""".
format(sparkVersionFromPySparkling,
sparkVersionFromPySpark))
return version
def getVersion():
here = path.abspath(path.dirname(__file__))
if '.zip' in here:
with zipfile.ZipFile(here[:-len("ai/h2o/sparkling/")],
'r') as archive:
version = archive.read('ai/h2o/sparkling/version.txt').decode(
'utf-8').strip()
else:
with open(path.join(here, 'version.txt'), encoding='utf-8') as f:
version = f.read().strip()
pySparklingVersionComponents = VersionComponents.parseFromSparklingWaterVersion(
version)
pySparkVersionComponents = VersionComponents.parseFromPySparkVersion(
pyspark.__version__)
sparkVersionFromPySparkling = pySparklingVersionComponents.sparkMajorMinorVersion
sparkVersionFromPySpark = pySparkVersionComponents.sparkMajorMinorVersion
if not Initializer.isRunningViaDBCConnect() and not (
sparkVersionFromPySpark == sparkVersionFromPySparkling):
raise Exception("""
You are using PySparkling for Spark {}, but your PySpark is of version {}.
Please make sure Spark and PySparkling versions are compatible.""".
format(sparkVersionFromPySparkling,
sparkVersionFromPySpark))
if sparkVersionFromPySpark == "2.1":
warnings.warn(
"PySpark {} is deprecated and ".format(pyspark.__version__) +
"the support will be removed in the Sparkling Water version 3.34."
)
return version
def testVersionComponentsCanBeParsedFromRegularVersion():
version = "3.26.2-2.4"
components = VersionComponents.parseFromSparklingWaterVersion(version)
assert components.fullVersion == version
assert components.sparklingVersion == "3.26.2"
assert components.sparklingMajorVersion == "3.26"
assert components.sparklingMinorVersion == "2"
assert components.nightlyVersion is None
assert components.sparkMajorMinorVersion == "2.4"
def testParseSparkVersionWithComplexSuffix():
version = "2.4.1.a.b.c"
components = VersionComponents.parseFromPySparkVersion(version)
assert components.fullVersion == version
assert components.sparkMajorMinorVersion == "2.4"
assert components.sparkMajorVersion == "2"
assert components.sparkMinorVersion == "4"
assert components.sparkPatchVersion == "1"
assert components.suffix == ".a.b.c"
def testParseSparkVersionWithHadoopSuffix():
version = "2.4.1+hadoop2.6"
components = VersionComponents.parseFromPySparkVersion(version)
assert components.fullVersion == version
assert components.sparkMajorMinorVersion == "2.4"
assert components.sparkMajorVersion == "2"
assert components.sparkMinorVersion == "4"
assert components.sparkPatchVersion == "1"
assert components.suffix == "+hadoop2.6"
def testParseSparkVersionWithBuildNumber():
version = "2.4.1.dev0"
components = VersionComponents.parseFromPySparkVersion(version)
assert components.fullVersion == version
assert components.sparkMajorMinorVersion == "2.4"
assert components.sparkMajorVersion == "2"
assert components.sparkMinorVersion == "4"
assert components.sparkPatchVersion == "1"
assert components.suffix == ".dev0"
def testParseSparkVersion():
version = "2.4.1"
components = VersionComponents.parseFromPySparkVersion(version)
assert components.fullVersion == version
assert components.sparkMajorMinorVersion == "2.4"
assert components.sparkMajorVersion == "2"
assert components.sparkMinorVersion == "4"
assert components.sparkPatchVersion == "1"
assert components.suffix is None
def testVersionComponentsCanBeParsedFromNightlyVersion():
version = "3.28.1-14-2.3"
components = VersionComponents.parseFromSparklingWaterVersion(version)
assert components.fullVersion == version
assert components.sparklingVersion == "3.28.1"
assert components.sparklingMajorVersion == "3.28"
assert components.sparklingMinorVersion == "1"
assert components.nightlyVersion == "14"
assert components.sparkMajorMinorVersion == "2.3"