def test_spark_script_in_standalone_lib():
# We must run this test in a subprocess in order for it to have an isolated
# Spark session
SCRIPT_PATH = testutil.get_fixture_path(
'test_spark_script_in_standalone_lib.py')
from oarphpy import util
out = util.run_cmd(sys.executable + ' ' + SCRIPT_PATH, collect=True)
assert b'test_spark_script_in_standalone_lib success!' in out
def test_spark_with_custom_library_in_notebook():
pytest.importorskip("jupyter")
import re
from oarphpy import util
NB_PATH = testutil.get_fixture_path(
'test_spark_ships_custom_library_in_notebook.ipynb')
# We need to fork off nbconvert to run the test
TEST_CMD = """
cd /tmp &&
jupyter-nbconvert --ExecutePreprocessor.timeout=3600 \
--to notebook --execute --output /tmp/out \
{notebook_path}
""".format(notebook_path=NB_PATH)
out = util.run_cmd(TEST_CMD, collect=True)
assert re.search('Writing .* bytes to /tmp/out.ipynb', out.decode())
import sys
from oarphpy import util
TEST_TEMPDIR_ROOT = '/tmp/test_spark_script_in_standalone_lib'
# First create a clean dir and a custom python library
my_lib_root = os.path.join(TEST_TEMPDIR_ROOT, 'my_lib_root')
util.cleandir(my_lib_root)
CREATE_LIB_SCRIPT = """
mkdir -p {src_root} &&
mkdir -p {src_root}/mylib &&
touch {src_root}/mylib/__init__.py
""".format(src_root=my_lib_root)
util.run_cmd(CREATE_LIB_SCRIPT)
with open(os.path.join(my_lib_root, 'mylib', 'util.py'), 'w') as f:
f.write(UTIL_PY_SRC)
with open(os.path.join(my_lib_root, 'mylib', 'prog.py'), 'w') as f:
f.write(PROG_PY_SRC)
# Make sure the custom library works
TEST_CMD = """
cd {src_root} &&
{python} -c 'from mylib.util import const_e; print(const_e)'
""".format(src_root=my_lib_root, python=sys.executable)
out = util.run_cmd(TEST_CMD, collect=True)
assert out == b'2.718\n', "Check contents of %s" % (my_lib_root, )
# Now test the embedded spark program