def test_invalid_reader_pandas_datasource(tmp_path_factory): basepath = str( tmp_path_factory.mktemp("test_invalid_reader_pandas_datasource")) datasource = PandasDatasource( "mypandassource", batch_kwargs_generators={ "subdir_reader": { "class_name": "SubdirReaderBatchKwargsGenerator", "base_directory": basepath, } }, ) with open(os.path.join(basepath, "idonotlooklikeacsvbutiam.notrecognized"), "w") as newfile: newfile.write("a,b\n1,2\n3,4\n") with pytest.raises(BatchKwargsError) as exc: datasource.get_batch( batch_kwargs={ "path": os.path.join(basepath, "idonotlooklikeacsvbutiam.notrecognized") }) assert "Unable to determine reader for path" in exc.value.message with pytest.raises(BatchKwargsError) as exc: datasource.get_batch( batch_kwargs={ "path": os.path.join(basepath, "idonotlooklikeacsvbutiam.notrecognized"), "reader_method": "blarg", }) assert "Unknown reader method: blarg" in exc.value.message batch = datasource.get_batch( batch_kwargs={ "path": os.path.join(basepath, "idonotlooklikeacsvbutiam.notrecognized"), "reader_method": "read_csv", "reader_options": { "header": 0 }, }) assert batch.data["a"][0] == 1
def test_read_limit(test_folder_connection_path_csv): datasource = PandasDatasource( "PandasCSV", batch_kwargs_generators={ "subdir_reader": { "class_name": "SubdirReaderBatchKwargsGenerator", "base_directory": test_folder_connection_path_csv, } }, ) batch_kwargs = PathBatchKwargs({ "path": os.path.join(str(test_folder_connection_path_csv), "test.csv"), # "reader_options": {"sep": ",", "header": 0, "index_col": 0}, "reader_options": { "sep": "," }, }) nested_update(batch_kwargs, datasource.process_batch_parameters(limit=1)) batch = datasource.get_batch(batch_kwargs=batch_kwargs) assert isinstance(batch, Batch) dataset = batch.data assert (dataset["col_1"] == [1]).all() assert len(dataset) == 1 # A datasource should always return an object with a typed batch_id assert isinstance(batch.batch_kwargs, PathBatchKwargs) assert isinstance(batch.batch_markers, BatchMarkers)
def test_invalid_reader_pandas_datasource(tmp_path_factory): basepath = str( tmp_path_factory.mktemp("test_invalid_reader_pandas_datasource")) datasource = PandasDatasource('mypandassource', base_directory=basepath) with open(os.path.join(basepath, "idonotlooklikeacsvbutiam.notrecognized"), "w") as newfile: newfile.write("a,b\n1,2\n3,4\n") with pytest.raises(BatchKwargsError) as exc: datasource.get_batch("idonotlooklikeacsvbutiam.notrecognized", expectation_suite_name="default", batch_kwargs={ "path": os.path.join( basepath, "idonotlooklikeacsvbutiam.notrecognized") }) assert "Unable to determine reader for path" in exc.message with pytest.raises(BatchKwargsError) as exc: datasource.get_batch("idonotlooklikeacsvbutiam.notrecognized", expectation_suite_name="default", batch_kwargs={ "path": os.path.join( basepath, "idonotlooklikeacsvbutiam.notrecognized") }, reader_method="blarg") assert "Unknown reader method: blarg" in exc.message dataset = datasource.get_batch( "idonotlooklikeacsvbutiam.notrecognized", expectation_suite_name="default", batch_kwargs={ "path": os.path.join(basepath, "idonotlooklikeacsvbutiam.notrecognized") }, reader_method="csv", header=0) assert dataset["a"][0] == 1
def test_standalone_pandas_datasource(test_folder_connection_path): datasource = PandasDatasource('PandasCSV', base_directory=test_folder_connection_path) assert datasource.get_available_data_asset_names() == {"default": {"test"}} manual_batch_kwargs = datasource.build_batch_kwargs(os.path.join(str(test_folder_connection_path), "test.csv")) # Get the default (subdir_path) generator generator = datasource.get_generator() auto_batch_kwargs = generator.yield_batch_kwargs("test") assert manual_batch_kwargs["path"] == auto_batch_kwargs["path"] # Include some extra kwargs... dataset = datasource.get_batch("test", batch_kwargs=auto_batch_kwargs, sep=",", header=0, index_col=0) assert isinstance(dataset, PandasDataset) assert (dataset["col_1"] == [1, 2, 3, 4, 5]).all()
def test_pandas_datasource_processes_dataset_options( test_folder_connection_path): datasource = PandasDatasource( "PandasCSV", batch_kwargs_generators={ "subdir_reader": { "class_name": "SubdirReaderBatchKwargsGenerator", "base_directory": test_folder_connection_path, } }, ) batch_kwargs = datasource.build_batch_kwargs("subdir_reader", name="test") batch_kwargs["dataset_options"] = {"caching": False} batch = datasource.get_batch(batch_kwargs) validator = Validator(batch, ExpectationSuite(expectation_suite_name="foo")) dataset = validator.get_dataset() assert dataset.caching is False
def test_standalone_pandas_datasource(test_folder_connection_path): datasource = PandasDatasource( "PandasCSV", batch_kwargs_generators={ "subdir_reader": { "class_name": "SubdirReaderBatchKwargsGenerator", "base_directory": test_folder_connection_path, } }, ) assert datasource.get_available_data_asset_names() == { "subdir_reader": { "names": [("test", "file")], "is_complete_list": True } } manual_batch_kwargs = PathBatchKwargs( path=os.path.join(str(test_folder_connection_path), "test.csv")) generator = datasource.get_batch_kwargs_generator("subdir_reader") auto_batch_kwargs = generator.yield_batch_kwargs("test") assert manual_batch_kwargs["path"] == auto_batch_kwargs["path"] # Include some extra kwargs... auto_batch_kwargs.update( {"reader_options": { "sep": ",", "header": 0, "index_col": 0 }}) batch = datasource.get_batch(batch_kwargs=auto_batch_kwargs) assert isinstance(batch, Batch) dataset = batch.data assert (dataset["col_1"] == [1, 2, 3, 4, 5]).all() assert len(dataset) == 5 # A datasource should always return an object with a typed batch_id assert isinstance(batch.batch_kwargs, PathBatchKwargs) assert isinstance(batch.batch_markers, BatchMarkers)