def test_spark_existing(): sc = SparkContext() station.start(sc) assert station.mode() == 'spark' assert isinstance(station.engine(), SparkContext) assert station.engine().parallelize([1,2,3]).count() == 3 station.stop()
def test_spark_opts(): station.start(spark=True, opts={'master': 'local', 'appName': 'hello'}) assert station.mode() == 'spark' assert isinstance(station.engine(), SparkContext) assert station.engine().master == 'local' assert station.engine().appName == 'hello' assert station.engine().parallelize([1,2,3]).count() == 3 station.stop()
def eng(request): engine = request.config.getoption("--engine") if engine == 'local': return None if engine == 'spark': station.start(spark=True) return station.engine()
def test_spark_context(): with station.start(spark=True): mode = station.mode() n = station.engine().parallelize([1,2,3]).count() assert mode == 'spark' assert n == 3 assert station.mode() == 'local'
def test_local(): station.start() assert station.engine() is None assert station.mode() == 'local'
def test_spark(): station.start(spark=True) assert station.mode() == 'spark' assert isinstance(station.engine(), SparkContext) assert station.engine().parallelize([1,2,3]).count() == 3 station.stop()