def test_invalid_reader_pandas_datasource(tmp_path_factory):
basepath = str(
tmp_path_factory.mktemp("test_invalid_reader_pandas_datasource"))
datasource = PandasDatasource(
"mypandassource",
batch_kwargs_generators={
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": basepath,
}
},
)
with open(os.path.join(basepath, "idonotlooklikeacsvbutiam.notrecognized"),
"w") as newfile:
newfile.write("a,b\n1,2\n3,4\n")
with pytest.raises(BatchKwargsError) as exc:
datasource.get_batch(
batch_kwargs={
"path":
os.path.join(basepath,
"idonotlooklikeacsvbutiam.notrecognized")
})
assert "Unable to determine reader for path" in exc.value.message
with pytest.raises(BatchKwargsError) as exc:
datasource.get_batch(
batch_kwargs={
"path":
os.path.join(basepath,
"idonotlooklikeacsvbutiam.notrecognized"),
"reader_method":
"blarg",
})
assert "Unknown reader method: blarg" in exc.value.message
batch = datasource.get_batch(
batch_kwargs={
"path":
os.path.join(basepath, "idonotlooklikeacsvbutiam.notrecognized"),
"reader_method":
"read_csv",
"reader_options": {
"header": 0
},
})
assert batch.data["a"][0] == 1
def test_read_limit(test_folder_connection_path_csv):
datasource = PandasDatasource(
"PandasCSV",
batch_kwargs_generators={
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": test_folder_connection_path_csv,
}
},
)
batch_kwargs = PathBatchKwargs({
"path":
os.path.join(str(test_folder_connection_path_csv), "test.csv"),
# "reader_options": {"sep": ",", "header": 0, "index_col": 0},
"reader_options": {
"sep": ","
},
})
nested_update(batch_kwargs, datasource.process_batch_parameters(limit=1))
batch = datasource.get_batch(batch_kwargs=batch_kwargs)
assert isinstance(batch, Batch)
dataset = batch.data
assert (dataset["col_1"] == [1]).all()
assert len(dataset) == 1
# A datasource should always return an object with a typed batch_id
assert isinstance(batch.batch_kwargs, PathBatchKwargs)
assert isinstance(batch.batch_markers, BatchMarkers)
def test_invalid_reader_pandas_datasource(tmp_path_factory):
basepath = str(
tmp_path_factory.mktemp("test_invalid_reader_pandas_datasource"))
datasource = PandasDatasource('mypandassource', base_directory=basepath)
with open(os.path.join(basepath, "idonotlooklikeacsvbutiam.notrecognized"),
"w") as newfile:
newfile.write("a,b\n1,2\n3,4\n")
with pytest.raises(BatchKwargsError) as exc:
datasource.get_batch("idonotlooklikeacsvbutiam.notrecognized",
expectation_suite_name="default",
batch_kwargs={
"path":
os.path.join(
basepath,
"idonotlooklikeacsvbutiam.notrecognized")
})
assert "Unable to determine reader for path" in exc.message
with pytest.raises(BatchKwargsError) as exc:
datasource.get_batch("idonotlooklikeacsvbutiam.notrecognized",
expectation_suite_name="default",
batch_kwargs={
"path":
os.path.join(
basepath,
"idonotlooklikeacsvbutiam.notrecognized")
},
reader_method="blarg")
assert "Unknown reader method: blarg" in exc.message
dataset = datasource.get_batch(
"idonotlooklikeacsvbutiam.notrecognized",
expectation_suite_name="default",
batch_kwargs={
"path":
os.path.join(basepath, "idonotlooklikeacsvbutiam.notrecognized")
},
reader_method="csv",
header=0)
assert dataset["a"][0] == 1
def test_standalone_pandas_datasource(test_folder_connection_path):
datasource = PandasDatasource('PandasCSV', base_directory=test_folder_connection_path)
assert datasource.get_available_data_asset_names() == {"default": {"test"}}
manual_batch_kwargs = datasource.build_batch_kwargs(os.path.join(str(test_folder_connection_path), "test.csv"))
# Get the default (subdir_path) generator
generator = datasource.get_generator()
auto_batch_kwargs = generator.yield_batch_kwargs("test")
assert manual_batch_kwargs["path"] == auto_batch_kwargs["path"]
# Include some extra kwargs...
dataset = datasource.get_batch("test", batch_kwargs=auto_batch_kwargs, sep=",", header=0, index_col=0)
assert isinstance(dataset, PandasDataset)
assert (dataset["col_1"] == [1, 2, 3, 4, 5]).all()
def test_pandas_datasource_processes_dataset_options(
test_folder_connection_path):
datasource = PandasDatasource(
"PandasCSV",
batch_kwargs_generators={
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": test_folder_connection_path,
}
},
)
batch_kwargs = datasource.build_batch_kwargs("subdir_reader", name="test")
batch_kwargs["dataset_options"] = {"caching": False}
batch = datasource.get_batch(batch_kwargs)
validator = Validator(batch,
ExpectationSuite(expectation_suite_name="foo"))
dataset = validator.get_dataset()
assert dataset.caching is False
def test_standalone_pandas_datasource(test_folder_connection_path):
datasource = PandasDatasource(
"PandasCSV",
batch_kwargs_generators={
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": test_folder_connection_path,
}
},
)
assert datasource.get_available_data_asset_names() == {
"subdir_reader": {
"names": [("test", "file")],
"is_complete_list": True
}
}
manual_batch_kwargs = PathBatchKwargs(
path=os.path.join(str(test_folder_connection_path), "test.csv"))
generator = datasource.get_batch_kwargs_generator("subdir_reader")
auto_batch_kwargs = generator.yield_batch_kwargs("test")
assert manual_batch_kwargs["path"] == auto_batch_kwargs["path"]
# Include some extra kwargs...
auto_batch_kwargs.update(
{"reader_options": {
"sep": ",",
"header": 0,
"index_col": 0
}})
batch = datasource.get_batch(batch_kwargs=auto_batch_kwargs)
assert isinstance(batch, Batch)
dataset = batch.data
assert (dataset["col_1"] == [1, 2, 3, 4, 5]).all()
assert len(dataset) == 5
# A datasource should always return an object with a typed batch_id
assert isinstance(batch.batch_kwargs, PathBatchKwargs)
assert isinstance(batch.batch_markers, BatchMarkers)