"Number of rows should match") self.assertEquals(len(df.columns), h2o_frame.ncol, "Number of columns should match") self.assertEquals(df.columns, h2o_frame.names, "Column names should match") class H2OConfTest(unittest.TestCase): @classmethod def setUpClass(cls): cls._sc = SparkContext(conf=test_utils.get_default_spark_conf().set( "spark.ext.h2o.cloud.name", "test-cloud")) test_utils.set_up_class(cls) h2o_conf = H2OConf(cls._sc).set_num_of_external_h2o_nodes(2) cls._hc = H2OContext.getOrCreate(cls._sc, h2o_conf) @classmethod def tearDownClass(cls): test_utils.tear_down_class(cls) # test passing h2o_conf to H2OContext def test_h2o_conf(self): self.assertEquals(self._hc.get_conf().cloud_name(), "test-cloud", "Configuration property cloud_name should match") if __name__ == '__main__': test_utils.run_tests([FrameTransformationsTest], file_name="py_unit_tests_report") #test_utils.run_tests([FrameTransformationsTest, H2OConfTest], file_name="py_unit_tests_report")
def main(testnames, debug): run_tests(TestChatBot, testnames, debug)
hc = self._hc rdd = self._sc.parallelize(["a","b","c"]) h2o_frame = hc.as_h2o_frame(rdd) df = hc.as_spark_frame(h2o_frame) self.assertEquals(df.count(), h2o_frame.nrow, "Number of rows should match") self.assertEquals(len(df.columns), h2o_frame.ncol, "Number of columns should match") self.assertEquals(df.columns,h2o_frame.names, "Column names should match") class H2OConfTest(unittest.TestCase): @classmethod def setUpClass(cls): cls._sc = SparkContext(conf = test_utils.get_default_spark_conf().set("spark.ext.h2o.cloud.name", "test-cloud")) test_utils.set_up_class(cls) h2o_conf = H2OConf(cls._sc) cls._hc = H2OContext.getOrCreate(cls._sc, h2o_conf) @classmethod def tearDownClass(cls): test_utils.tear_down_class(cls) # test passing h2o_conf to H2OContext def test_h2o_conf(self): self.assertEquals(self._hc.get_conf().cloud_name(),"test-cloud", "Configuration property cloud_name should match") if __name__ == '__main__': test_utils.run_tests(FrameTransformationsTest, file_name="py_unit_tests_report") test_utils.run_tests(H2OConfTest, file_name="py_h2oconf_unit_tests_report")
# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Integration tests for pySparkling for Spark running in local """ from integ_test_utils import IntegTestSuite import test_utils class LocalIntegTestSuite(IntegTestSuite): pass if __name__ == '__main__': test_utils.run_tests([LocalIntegTestSuite], file_name="py_integ_local_tests_report")
# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Integration tests for pySparkling for spark running in Standalone mode """ from integ_test_utils import IntegTestSuite import test_utils class StandaloneIntegTestSuite(IntegTestSuite): pass if __name__ == '__main__': test_utils.run_tests(StandaloneIntegTestSuite, file_name="py_integ_standalone_tests_report")
# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Integration tests for pySparkling for spark running in Standalone mode """ from integ_test_utils import IntegTestSuite import test_utils class StandaloneIntegTestSuite(IntegTestSuite): pass if __name__ == '__main__': test_utils.run_tests([StandaloneIntegTestSuite], file_name="py_integ_standalone_tests_report")
def main(testnames, debug): run_tests(TestLUIS, testnames, debug)
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ Integration tests for pySparkling for Spark running in YARN mode """ from integ_test_utils import IntegTestSuite import test_utils class YarnIntegTestSuite(IntegTestSuite): def test_chicago_crime(self): self.spark_master("yarn-client") # Configure YARN environment self.conf("spark.yarn.max.executor.failures", 1) # In fail of executor, fail the test self.conf("spark.executor.instances", 3) self.conf("spark.executor.memory", "2g") self.conf("spark.ext.h2o.port.base", 63331) self.conf("spark.driver.memory", "2g") self.launch("examples/scripts/ChicagoCrimeDemo.py") if __name__ == '__main__': test_utils.run_tests(YarnIntegTestSuite, file_name="py_integ_yarn_tests_report")
hf = hc.as_h2o_frame(df) # Modify H2O frame - this should invalidate internal cache hf['c3'] = 3 # Now try to convert modified H2O frame back to Spark data frame dfe = hc.as_spark_frame(hf) self.assertEquals(dfe.count(), len(data), "Number of rows should match") self.assertEquals(len(dfe.columns), 3, "Number of columns should match") self.assertEquals(dfe.collect(), [Row(c1=1, c2='first', c3=3), Row(c1=2, c2='second', c3=3)]) class H2OConfTest(unittest.TestCase): @classmethod def setUpClass(cls): cls._spark = SparkSession.builder.config(conf = test_utils.get_default_spark_conf().set("spark.ext.h2o.cloud.name", "test-cloud")).getOrCreate() test_utils.set_up_class(cls) h2o_conf = H2OConf(cls._spark).set_num_of_external_h2o_nodes(2) cls._hc = H2OContext.getOrCreate(cls._spark, h2o_conf) @classmethod def tearDownClass(cls): test_utils.tear_down_class(cls) # test passing h2o_conf to H2OContext def test_h2o_conf(self): self.assertEquals(self._hc.get_conf().cloud_name(),"test-cloud", "Configuration property cloud_name should match") if __name__ == '__main__': test_utils.run_tests([FrameTransformationsTest], file_name="py_unit_tests_report") #test_utils.run_tests([FrameTransformationsTest, H2OConfTest], file_name="py_unit_tests_report")