Example #1
0
 def setUpClass(cls):
     ReusedPySparkTestCase.setUpClass()
     cls.tempdir = tempfile.NamedTemporaryFile(delete=False)
     os.unlink(cls.tempdir.name)
     cls.sqlCtx = SQLContext(cls.sc)
     cls.testData = [Row(key=i, value=str(i)) for i in range(100)]
     rdd = cls.sc.parallelize(cls.testData)
     cls.df = rdd.toDF()
Example #2
0
 def setUpClass(cls):
     ReusedPySparkTestCase.setUpClass()
     cls.tempdir = tempfile.NamedTemporaryFile(delete=False)
     os.unlink(cls.tempdir.name)
     print "type", type(cls.sc)
     print "type", type(cls.sc._jsc)
     _scala_HiveContext =\
         cls.sc._jvm.org.apache.spark.sql.hive.test.TestHiveContext(cls.sc._jsc.sc())
     cls.sqlCtx = HiveContext(cls.sc, _scala_HiveContext)
     cls.testData = [Row(key=i, value=str(i)) for i in range(100)]
     cls.df = cls.sc.parallelize(cls.testData).toDF()
Example #3
0
 def setUpClass(cls):
     ReusedPySparkTestCase.setUpClass()
     cls.tempdir = tempfile.NamedTemporaryFile(delete=False)
     try:
         cls.sc._jvm.org.apache.hadoop.hive.conf.HiveConf()
     except py4j.protocol.Py4JError:
         cls.sqlCtx = None
         return
     os.unlink(cls.tempdir.name)
     _scala_HiveContext = cls.sc._jvm.org.apache.spark.sql.hive.test.TestHiveContext(cls.sc._jsc.sc())
     cls.sqlCtx = HiveContext(cls.sc, _scala_HiveContext)
     cls.testData = [Row(key=i, value=str(i)) for i in range(100)]
     cls.df = cls.sc.parallelize(cls.testData).toDF()
Example #4
0
 def setUpClass(cls):
     ReusedPySparkTestCase.setUpClass()
     cls.tempdir = tempfile.NamedTemporaryFile(delete=False)
     try:
         cls.sc._jvm.org.apache.hadoop.hive.conf.HiveConf()
     except py4j.protocol.Py4JError:
         cls.tearDownClass()
         raise unittest.SkipTest("Hive is not available")
     except TypeError:
         cls.tearDownClass()
         raise unittest.SkipTest("Hive is not available")
     os.unlink(cls.tempdir.name)
     _scala_HiveContext =\
         cls.sc._jvm.org.apache.spark.sql.hive.test.TestHiveContext(cls.sc._jsc.sc())
     cls.sqlCtx = HiveContext(cls.sc, _scala_HiveContext)
     cls.testData = [Row(key=i, value=str(i)) for i in range(100)]
     cls.df = cls.sc.parallelize(cls.testData).toDF()
Example #5
0
 def tearDownClass(cls):
     ReusedPySparkTestCase.tearDownClass()
     shutil.rmtree(cls.tempdir.name, ignore_errors=True)
Example #6
0
 def setUpClass(cls):
     ReusedPySparkTestCase.setUpClass()
     cls.tempdir = tempfile.mkdtemp()
Example #7
0
 def tearDownClass(cls):
     ReusedPySparkTestCase.tearDownClass()
     shutil.rmtree(cls.tempdir)
Example #8
0
 def setUpClass(cls):
     ReusedPySparkTestCase.setUpClass()
     cls.spark = SparkSession(cls.sc)
     cls.tempdir = tempfile.mkdtemp()
Example #9
0
 def tearDownClass(cls):
     ReusedPySparkTestCase.tearDownClass()
     cls.spark.stop()
Example #10
0
 def setUpClass(cls):
     ReusedPySparkTestCase.setUpClass()
     cls.spark = SparkSession(cls.sc)
Example #11
0
 def tearDownClass(cls):
     PySparkTestCase.tearDownClass()
     cls.spark.stop()
Example #12
0
 def setUpClass(cls):
     PySparkTestCase.setUpClass()
     cls.spark = SparkSession(cls.sc)