"Number of rows should match")
self.assertEquals(len(df.columns), h2o_frame.ncol,
"Number of columns should match")
self.assertEquals(df.columns, h2o_frame.names,
"Column names should match")
class H2OConfTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._sc = SparkContext(conf=test_utils.get_default_spark_conf().set(
"spark.ext.h2o.cloud.name", "test-cloud"))
test_utils.set_up_class(cls)
h2o_conf = H2OConf(cls._sc).set_num_of_external_h2o_nodes(2)
cls._hc = H2OContext.getOrCreate(cls._sc, h2o_conf)
@classmethod
def tearDownClass(cls):
test_utils.tear_down_class(cls)
# test passing h2o_conf to H2OContext
def test_h2o_conf(self):
self.assertEquals(self._hc.get_conf().cloud_name(), "test-cloud",
"Configuration property cloud_name should match")
if __name__ == '__main__':
test_utils.run_tests([FrameTransformationsTest],
file_name="py_unit_tests_report")
#test_utils.run_tests([FrameTransformationsTest, H2OConfTest], file_name="py_unit_tests_report")
def main(testnames, debug):
run_tests(TestChatBot, testnames, debug)
hc = self._hc
rdd = self._sc.parallelize(["a","b","c"])
h2o_frame = hc.as_h2o_frame(rdd)
df = hc.as_spark_frame(h2o_frame)
self.assertEquals(df.count(), h2o_frame.nrow, "Number of rows should match")
self.assertEquals(len(df.columns), h2o_frame.ncol, "Number of columns should match")
self.assertEquals(df.columns,h2o_frame.names, "Column names should match")
class H2OConfTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._sc = SparkContext(conf = test_utils.get_default_spark_conf().set("spark.ext.h2o.cloud.name", "test-cloud"))
test_utils.set_up_class(cls)
h2o_conf = H2OConf(cls._sc)
cls._hc = H2OContext.getOrCreate(cls._sc, h2o_conf)
@classmethod
def tearDownClass(cls):
test_utils.tear_down_class(cls)
# test passing h2o_conf to H2OContext
def test_h2o_conf(self):
self.assertEquals(self._hc.get_conf().cloud_name(),"test-cloud", "Configuration property cloud_name should match")
if __name__ == '__main__':
test_utils.run_tests(FrameTransformationsTest, file_name="py_unit_tests_report")
test_utils.run_tests(H2OConfTest, file_name="py_h2oconf_unit_tests_report")
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Integration tests for pySparkling for Spark running in local
"""
from integ_test_utils import IntegTestSuite
import test_utils
class LocalIntegTestSuite(IntegTestSuite):
pass
if __name__ == '__main__':
test_utils.run_tests([LocalIntegTestSuite], file_name="py_integ_local_tests_report")
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Integration tests for pySparkling for spark running in Standalone mode
"""
from integ_test_utils import IntegTestSuite
import test_utils
class StandaloneIntegTestSuite(IntegTestSuite):
pass
if __name__ == '__main__':
test_utils.run_tests(StandaloneIntegTestSuite,
file_name="py_integ_standalone_tests_report")
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Integration tests for pySparkling for spark running in Standalone mode
"""
from integ_test_utils import IntegTestSuite
import test_utils
class StandaloneIntegTestSuite(IntegTestSuite):
pass
if __name__ == '__main__':
test_utils.run_tests([StandaloneIntegTestSuite], file_name="py_integ_standalone_tests_report")
def main(testnames, debug):
run_tests(TestLUIS, testnames, debug)
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Integration tests for pySparkling for Spark running in YARN mode
"""
from integ_test_utils import IntegTestSuite
import test_utils
class YarnIntegTestSuite(IntegTestSuite):
def test_chicago_crime(self):
self.spark_master("yarn-client")
# Configure YARN environment
self.conf("spark.yarn.max.executor.failures", 1) # In fail of executor, fail the test
self.conf("spark.executor.instances", 3)
self.conf("spark.executor.memory", "2g")
self.conf("spark.ext.h2o.port.base", 63331)
self.conf("spark.driver.memory", "2g")
self.launch("examples/scripts/ChicagoCrimeDemo.py")
if __name__ == '__main__':
test_utils.run_tests(YarnIntegTestSuite, file_name="py_integ_yarn_tests_report")
hf = hc.as_h2o_frame(df)
# Modify H2O frame - this should invalidate internal cache
hf['c3'] = 3
# Now try to convert modified H2O frame back to Spark data frame
dfe = hc.as_spark_frame(hf)
self.assertEquals(dfe.count(), len(data), "Number of rows should match")
self.assertEquals(len(dfe.columns), 3, "Number of columns should match")
self.assertEquals(dfe.collect(), [Row(c1=1, c2='first', c3=3), Row(c1=2, c2='second', c3=3)])
class H2OConfTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._spark = SparkSession.builder.config(conf = test_utils.get_default_spark_conf().set("spark.ext.h2o.cloud.name", "test-cloud")).getOrCreate()
test_utils.set_up_class(cls)
h2o_conf = H2OConf(cls._spark).set_num_of_external_h2o_nodes(2)
cls._hc = H2OContext.getOrCreate(cls._spark, h2o_conf)
@classmethod
def tearDownClass(cls):
test_utils.tear_down_class(cls)
# test passing h2o_conf to H2OContext
def test_h2o_conf(self):
self.assertEquals(self._hc.get_conf().cloud_name(),"test-cloud", "Configuration property cloud_name should match")
if __name__ == '__main__':
test_utils.run_tests([FrameTransformationsTest], file_name="py_unit_tests_report")
#test_utils.run_tests([FrameTransformationsTest, H2OConfTest], file_name="py_unit_tests_report")