def test_exercise_3():
modified_notebook_path = os.path.join(local_notebooks_dir,
"exercise_3_test.ipynb")
nb = nbformat.read(
os.path.join(local_notebooks_dir, "exercise_3.ipynb"),
as_version=nbformat.NO_CONVERT,
)
user_mad_implementation = """PandasQueryCompiler.sq_mad_custom = TreeReduce.register(lambda cell_value, **kwargs: cell_value ** 2,
pandas.DataFrame.mad)
def sq_mad_func(self, axis=None, skipna=True, level=None, **kwargs):
if axis is None:
axis = 0
return self._reduce_dimension(
self._query_compiler.sq_mad_custom(
axis=axis, skipna=skipna, level=level, **kwargs
)
)
pd.DataFrame.sq_mad_custom = sq_mad_func
modin_mad_custom = df.sq_mad_custom()
"""
_replace_str(nb, "modin_mad_custom = ...", user_mad_implementation)
nbformat.write(nb, modified_notebook_path)
_execute_notebook(modified_notebook_path)
def test_exercise_3():
modified_notebook_path = os.path.join(local_notebooks_dir, "exercise_3_test.ipynb")
nb = nbformat.read(
os.path.join(local_notebooks_dir, "exercise_3.ipynb"),
as_version=nbformat.NO_CONVERT,
)
nbformat.write(nb, modified_notebook_path)
_execute_notebook(modified_notebook_path)
def test_exercise_1():
modified_notebook_path = os.path.join(local_notebooks_dir, "exercise_1_test.ipynb")
nb = nbformat.read(
os.path.join(local_notebooks_dir, "exercise_1.ipynb"),
as_version=nbformat.NO_CONVERT,
)
_replace_str(nb, "import pandas as pd", "import modin.pandas as pd")
nbformat.write(nb, modified_notebook_path)
_execute_notebook(modified_notebook_path)
def test_exercise_4():
modified_notebook_path = os.path.join(local_notebooks_dir,
"exercise_4_test.ipynb")
nb = nbformat.read(
os.path.join(local_notebooks_dir, "exercise_4.ipynb"),
as_version=nbformat.NO_CONVERT,
)
s3_path_cell = f's3_path = "{test_dataset_path}"\n' + download_taxi_dataset
_replace_str(
nb,
's3_path = "s3://dask-data/nyc-taxi/2015/yellow_tripdata_2015-01.csv"',
s3_path_cell,
)
nbformat.write(nb, modified_notebook_path)
_execute_notebook(modified_notebook_path)
def test_exercise_2():
modified_notebook_path = os.path.join(local_notebooks_dir,
"exercise_2_test.ipynb")
nb = nbformat.read(
os.path.join(local_notebooks_dir, "exercise_2.ipynb"),
as_version=nbformat.NO_CONVERT,
)
_replace_str(
nb,
'path = "s3://dask-data/nyc-taxi/2015/yellow_tripdata_2015-01.csv"',
'# path = "s3://dask-data/nyc-taxi/2015/yellow_tripdata_2015-01.csv"',
)
new_optional_cell = f'path = "{test_dataset_path}"\n' + download_taxi_dataset
optional_cell_idx = _find_code_cell_idx(
nb, "[Optional] Download data locally.")
nb["cells"][optional_cell_idx]["source"] = new_optional_cell
nbformat.write(nb, modified_notebook_path)
_execute_notebook(modified_notebook_path)