def tc(request):
global global_tc
with lock:
if global_tc is None:
from sparktk import TkContext
from sparktk import create_sc
from sparktk.tests import utils
import daaltk
#from sparktk.loggers import loggers
#loggers.set("d", "sparktk.sparkconf")
# Get path to sparktk jars from SPARKTK_HOME
if os.environ.has_key('SPARKTK_HOME'):
sparktk_dir = os.environ.get('SPARKTK_HOME', None)
else:
raise RuntimeError("SPARKTK_HOME must be defined.")
sc = create_sc(
other_libs=[daaltk],
master='local[2]',
sparktk_home=sparktk_dir,
app_name="pytest-pyspark-local-testing",
extra_conf_dict={"spark.hadoop.fs.default.name": "file:///"})
request.addfinalizer(lambda: sc.stop())
global_tc = TkContext(sc,
other_libs=[daaltk],
sparktk_home=sparktk_dir)
global_tc.testing = utils
return global_tc
def tc(request):
global global_tc
with lock:
if global_tc is None:
from sparktk import TkContext
from sparktk import create_sc
from sparktk.tests import utils
#from sparktk.loggers import loggers
#loggers.set("d", "sparktk.sparkconf")
sc = create_sc(master='local[2]',
app_name="pytest-pyspark-local-testing",
extra_conf_dict={"spark.hadoop.fs.default.name": "file:///"})
request.addfinalizer(lambda: sc.stop())
global_tc = TkContext(sc)
global_tc.testing = utils
return global_tc
def tc(request):
global global_tc
with lock:
if global_tc is None:
from sparktk import TkContext
from sparktk import create_sc
from sparktk.tests import utils
#from sparktk.loggers import loggers
#loggers.set("d", "sparktk.sparkconf")
sc = create_sc(master='local[2]',
app_name="pytest-pyspark-local-testing",
extra_conf={"spark.hadoop.fs.default.name": "file:///"})
request.addfinalizer(lambda: sc.stop())
global_tc = TkContext(sc)
global_tc.testing = utils
return global_tc