Exemplo n.º 1
0
                          "Number of rows should match")
        self.assertEquals(len(df.columns), h2o_frame.ncol,
                          "Number of columns should match")
        self.assertEquals(df.columns, h2o_frame.names,
                          "Column names should match")


class H2OConfTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls._sc = SparkContext(conf=test_utils.get_default_spark_conf().set(
            "spark.ext.h2o.cloud.name", "test-cloud"))
        test_utils.set_up_class(cls)
        h2o_conf = H2OConf(cls._sc).set_num_of_external_h2o_nodes(2)
        cls._hc = H2OContext.getOrCreate(cls._sc, h2o_conf)

    @classmethod
    def tearDownClass(cls):
        test_utils.tear_down_class(cls)

    # test passing h2o_conf to H2OContext
    def test_h2o_conf(self):
        self.assertEquals(self._hc.get_conf().cloud_name(), "test-cloud",
                          "Configuration property cloud_name should match")


if __name__ == '__main__':
    test_utils.run_tests([FrameTransformationsTest],
                         file_name="py_unit_tests_report")
    #test_utils.run_tests([FrameTransformationsTest, H2OConfTest], file_name="py_unit_tests_report")
Exemplo n.º 2
0
def main(testnames, debug):
    run_tests(TestChatBot, testnames, debug)
Exemplo n.º 3
0
        hc = self._hc
        rdd = self._sc.parallelize(["a","b","c"])
        h2o_frame = hc.as_h2o_frame(rdd)
        df = hc.as_spark_frame(h2o_frame)
        self.assertEquals(df.count(), h2o_frame.nrow, "Number of rows should match")
        self.assertEquals(len(df.columns), h2o_frame.ncol, "Number of columns should match")
        self.assertEquals(df.columns,h2o_frame.names, "Column names should match")

class H2OConfTest(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls._sc = SparkContext(conf = test_utils.get_default_spark_conf().set("spark.ext.h2o.cloud.name", "test-cloud"))
        test_utils.set_up_class(cls)
        h2o_conf = H2OConf(cls._sc)
        cls._hc = H2OContext.getOrCreate(cls._sc, h2o_conf)

    @classmethod
    def tearDownClass(cls):
        test_utils.tear_down_class(cls)

    # test passing h2o_conf to H2OContext
    def test_h2o_conf(self):
        self.assertEquals(self._hc.get_conf().cloud_name(),"test-cloud", "Configuration property cloud_name should match")


if __name__ == '__main__':
    test_utils.run_tests(FrameTransformationsTest, file_name="py_unit_tests_report")
    test_utils.run_tests(H2OConfTest, file_name="py_h2oconf_unit_tests_report")

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Integration tests for pySparkling for Spark running in local
"""
from integ_test_utils import IntegTestSuite
import test_utils

class LocalIntegTestSuite(IntegTestSuite):
    pass


if __name__ == '__main__':
    test_utils.run_tests([LocalIntegTestSuite], file_name="py_integ_local_tests_report")
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
 Integration tests for pySparkling for spark running in Standalone mode
"""
from integ_test_utils import IntegTestSuite
import test_utils


class StandaloneIntegTestSuite(IntegTestSuite):
    pass


if __name__ == '__main__':
    test_utils.run_tests(StandaloneIntegTestSuite,
                         file_name="py_integ_standalone_tests_report")
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
 Integration tests for pySparkling for spark running in Standalone mode
"""
from integ_test_utils import IntegTestSuite
import test_utils

class StandaloneIntegTestSuite(IntegTestSuite):
    pass


if __name__ == '__main__':
    test_utils.run_tests([StandaloneIntegTestSuite], file_name="py_integ_standalone_tests_report")
Exemplo n.º 7
0
def main(testnames, debug):
    run_tests(TestLUIS, testnames, debug)
Exemplo n.º 8
0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Integration tests for pySparkling for Spark running in YARN mode
"""
from integ_test_utils import IntegTestSuite
import test_utils

class YarnIntegTestSuite(IntegTestSuite):

    def test_chicago_crime(self):
            self.spark_master("yarn-client")
            # Configure YARN environment
            self.conf("spark.yarn.max.executor.failures", 1) # In fail of executor, fail the test
            self.conf("spark.executor.instances", 3)
            self.conf("spark.executor.memory", "2g")
            self.conf("spark.ext.h2o.port.base", 63331)
            self.conf("spark.driver.memory", "2g")

            self.launch("examples/scripts/ChicagoCrimeDemo.py")


if __name__ == '__main__':
        test_utils.run_tests(YarnIntegTestSuite, file_name="py_integ_yarn_tests_report")
Exemplo n.º 9
0
        hf = hc.as_h2o_frame(df)
        # Modify H2O frame - this should invalidate internal cache
        hf['c3'] = 3
        # Now try to convert modified H2O frame back to Spark data frame
        dfe = hc.as_spark_frame(hf)
        self.assertEquals(dfe.count(), len(data), "Number of rows should match")
        self.assertEquals(len(dfe.columns), 3, "Number of columns should match")
        self.assertEquals(dfe.collect(), [Row(c1=1, c2='first', c3=3), Row(c1=2, c2='second', c3=3)])

class H2OConfTest(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls._spark = SparkSession.builder.config(conf = test_utils.get_default_spark_conf().set("spark.ext.h2o.cloud.name", "test-cloud")).getOrCreate()
        test_utils.set_up_class(cls)
        h2o_conf = H2OConf(cls._spark).set_num_of_external_h2o_nodes(2)
        cls._hc = H2OContext.getOrCreate(cls._spark, h2o_conf)

    @classmethod
    def tearDownClass(cls):
        test_utils.tear_down_class(cls)

    # test passing h2o_conf to H2OContext
    def test_h2o_conf(self):
        self.assertEquals(self._hc.get_conf().cloud_name(),"test-cloud", "Configuration property cloud_name should match")


if __name__ == '__main__':
    test_utils.run_tests([FrameTransformationsTest], file_name="py_unit_tests_report")
    #test_utils.run_tests([FrameTransformationsTest, H2OConfTest], file_name="py_unit_tests_report")