def test_sanitize_yaml_and_save_datasource_works_without_credentials(
sa,
empty_data_context,
):
context = empty_data_context
yaml_snippet = """
name: my_datasource
class_name: SimpleSqlalchemyDatasource
introspection:
whole_table:
data_asset_name_suffix: __whole_table
connection_string: sqlite://"""
assert len(context.list_datasources()) == 0
sanitize_yaml_and_save_datasource(context, yaml_snippet)
assert len(context.list_datasources()) == 1
assert context.list_datasources() == [{
"class_name": "SimpleSqlalchemyDatasource",
"connection_string": "sqlite://",
"introspection": {
"whole_table": {
"data_asset_name_suffix": "__whole_table"
}
},
"module_name": "great_expectations.datasource",
"name": "my_datasource",
}]
obs = context.config_variables
# remove the instance guid
obs.pop("instance_id")
assert obs == {}
def test_sanitize_yaml_and_save_datasource_raises_error_on_missing_datasource_name(
empty_data_context, ):
yaml_snippet = """
class_name: SimpleSqlalchemyDatasource
introspection:
whole_table:
data_asset_name_suffix: __whole_table
connection_string: sqlite://"""
with pytest.raises(ValueError):
sanitize_yaml_and_save_datasource(empty_data_context, yaml_snippet)
def test_sanitize_yaml_and_save_datasource_works_with_credentials(
sa,
empty_data_context,
):
context = empty_data_context
yaml_snippet = """
name: foo_datasource
class_name: SimpleSqlalchemyDatasource
credentials:
host: localhost
port: '5432'
username: user
password: pass
database: postgres
drivername: postgresql"""
assert len(context.list_datasources()) == 0
sanitize_yaml_and_save_datasource(context, yaml_snippet)
assert len(context.list_datasources()) == 1
assert context.list_datasources() == [{
"class_name": "SimpleSqlalchemyDatasource",
"credentials": {
"database": "postgres",
"drivername": "postgresql",
"host": "localhost",
"password": "******",
"port": "5432",
"username": "******",
},
"module_name": "great_expectations.datasource",
"name": "foo_datasource",
}]
obs = context.config_variables
# remove the instance guid
obs.pop("instance_id")
assert obs == {
"foo_datasource": {
"database": "postgres",
"drivername": "postgresql",
"host": "localhost",
"password": "******",
"port": "5432",
"username": "******",
}
}
def test_sanitize_yaml_and_save_datasource_does_not_overwrite_duplicate_when_overwrite_existing_is_false(
capsys,
empty_data_context,
):
context = empty_data_context
pandas_yaml_snippet = """
name: my_datasource
class_name: Datasource
execution_engine:
class_name: PandasExecutionEngine
"""
assert len(context.list_datasources()) == 0
sanitize_yaml_and_save_datasource(context,
pandas_yaml_snippet,
overwrite_existing=False)
assert len(context.list_datasources()) == 1
datasource_from_context = [{
"class_name": "Datasource",
"module_name": "great_expectations.datasource",
"name": "my_datasource",
"execution_engine": {
"class_name": "PandasExecutionEngine",
"module_name": "great_expectations.execution_engine",
},
}]
assert context.list_datasources() == datasource_from_context
sanitize_yaml_and_save_datasource(context,
pandas_yaml_snippet,
overwrite_existing=False)
assert len(context.list_datasources()) == 1
stdout = capsys.readouterr().out.strip()
assert (
'**WARNING** A Datasource named "my_datasource" already exists in this Data Context. The Datasource has *not* been saved. Please use a different name or set overwrite_existing=True if you want to overwrite!'
.strip() == stdout)
# retest with a different type of datasource with the same name
sql_yaml_snippet = """
name: my_datasource
class_name: SimpleSqlalchemyDatasource
introspection:
whole_table:
data_asset_name_suffix: __whole_table
connection_string: sqlite://"""
sanitize_yaml_and_save_datasource(context,
sql_yaml_snippet,
overwrite_existing=False)
assert len(context.list_datasources()) == 1
stdout = capsys.readouterr().out.strip()
assert (
'**WARNING** A Datasource named "my_datasource" already exists in this Data Context. The Datasource has *not* been saved. Please use a different name or set overwrite_existing=True if you want to overwrite!'
.strip() == stdout)
def test_sanitize_yaml_and_save_datasource_raises_error_on_empty_yaml(
empty_data_context, ):
with pytest.raises(ValueError):
sanitize_yaml_and_save_datasource(empty_data_context, "")
def test_sanitize_yaml_and_save_datasource_does_overwrite_duplicate_when_overwrite_existing_is_true(
sa,
capsys,
empty_data_context,
):
context = empty_data_context
pandas_yaml_snippet = """
name: my_datasource
class_name: Datasource
execution_engine:
class_name: PandasExecutionEngine
"""
assert len(context.list_datasources()) == 0
sanitize_yaml_and_save_datasource(context,
pandas_yaml_snippet,
overwrite_existing=True)
assert len(context.list_datasources()) == 1
pandas_datasource_from_context = [{
"class_name": "Datasource",
"module_name": "great_expectations.datasource",
"name": "my_datasource",
"execution_engine": {
"class_name": "PandasExecutionEngine",
"module_name": "great_expectations.execution_engine",
},
}]
assert context.list_datasources() == pandas_datasource_from_context
sanitize_yaml_and_save_datasource(context,
pandas_yaml_snippet,
overwrite_existing=True)
assert len(context.list_datasources()) == 1
assert context.list_datasources() == pandas_datasource_from_context
stdout = capsys.readouterr().out.strip()
assert stdout == ""
# retest with a different type of datasource with the same name
sql_yaml_snippet = """
name: my_datasource
class_name: SimpleSqlalchemyDatasource
introspection:
whole_table:
data_asset_name_suffix: __whole_table
connection_string: sqlite://"""
sanitize_yaml_and_save_datasource(context,
sql_yaml_snippet,
overwrite_existing=True)
assert len(context.list_datasources()) == 1
stdout = capsys.readouterr().out
assert stdout == ""
assert context.list_datasources() != pandas_datasource_from_context
sql_datasource_from_context = [{
"class_name": "SimpleSqlalchemyDatasource",
"connection_string": "sqlite://",
"introspection": {
"whole_table": {
"data_asset_name_suffix": "__whole_table"
}
},
"module_name": "great_expectations.datasource",
"name": "my_datasource",
}]
assert context.list_datasources() == sql_datasource_from_context
def test_sanitize_yaml_and_save_datasource_raises_error_on_non_string(
empty_data_context, ):
with pytest.raises(TypeError):
for bad_input in [3, {"a", "b"}]:
sanitize_yaml_and_save_datasource(empty_data_context, bad_input)