def test_import_v1_database_broken_csv_fields(self):
"""
Test that a database can be imported with broken schema.
https://github.com/apache/superset/pull/16756 renamed some fields, changing
the V1 schema. This test ensures that we can import databases that were
exported with the broken schema.
"""
broken_config = database_config.copy()
broken_config["allow_file_upload"] = broken_config.pop("allow_csv_upload")
broken_config["extra"] = {"schemas_allowed_for_file_upload": ["upload"]}
contents = {
"metadata.yaml": yaml.safe_dump(database_metadata_config),
"databases/imported_database.yaml": yaml.safe_dump(broken_config),
}
command = ImportDatabasesCommand(contents)
command.run()
database = (
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
)
assert database.allow_file_upload
assert database.allow_ctas
assert database.allow_cvas
assert not database.allow_run_async
assert database.cache_timeout is None
assert database.database_name == "imported_database"
assert database.expose_in_sqllab
assert database.extra == '{"schemas_allowed_for_file_upload": ["upload"]}'
assert database.sqlalchemy_uri == "sqlite:///test.db"
db.session.delete(database)
db.session.commit()
def test_import_v1_saved_queries_validation(self):
"""Test different validations applied when importing a saved query"""
# metadata.yaml must be present
contents = {
"databases/imported_database.yaml":
yaml.safe_dump(database_config),
"queries/imported_query.yaml":
yaml.safe_dump(saved_queries_config),
}
command = ImportSavedQueriesCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Missing metadata.yaml"
# version should be 1.0.0
contents["metadata.yaml"] = yaml.safe_dump({
"version":
"2.0.0",
"type":
"SavedQuery",
"timestamp":
"2021-03-30T20:37:54.791187+00:00",
})
command = ImportSavedQueriesCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Must be equal to 1.0.0."
# type should be a SavedQuery
contents["metadata.yaml"] = yaml.safe_dump(database_metadata_config)
command = ImportSavedQueriesCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing saved_queries"
assert excinfo.value.normalized_messages() == {
"metadata.yaml": {
"type": ["Must be equal to SavedQuery."]
}
}
# must also validate databases
broken_config = database_config.copy()
del broken_config["database_name"]
contents["metadata.yaml"] = yaml.safe_dump(
saved_queries_metadata_config)
contents["databases/imported_database.yaml"] = yaml.safe_dump(
broken_config)
command = ImportSavedQueriesCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing saved_queries"
assert excinfo.value.normalized_messages() == {
"databases/imported_database.yaml": {
"database_name": ["Missing data for required field."],
}
}
def test_import_v1_chart_validation(self):
"""Test different validations applied when importing a chart"""
# metadata.yaml must be present
contents = {
"databases/imported_database.yaml":
yaml.safe_dump(database_config),
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
"charts/imported_chart.yaml": yaml.safe_dump(chart_config),
}
command = ImportChartsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Missing metadata.yaml"
# version should be 1.0.0
contents["metadata.yaml"] = yaml.safe_dump({
"version":
"2.0.0",
"type":
"SqlaTable",
"timestamp":
"2020-11-04T21:27:44.423819+00:00",
})
command = ImportChartsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Must be equal to 1.0.0."
# type should be Slice
contents["metadata.yaml"] = yaml.safe_dump(database_metadata_config)
command = ImportChartsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing chart"
assert excinfo.value.normalized_messages() == {
"metadata.yaml": {
"type": ["Must be equal to Slice."]
}
}
# must also validate datasets and databases
broken_config = database_config.copy()
del broken_config["database_name"]
contents["metadata.yaml"] = yaml.safe_dump(chart_metadata_config)
contents["databases/imported_database.yaml"] = yaml.safe_dump(
broken_config)
command = ImportChartsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing chart"
assert excinfo.value.normalized_messages() == {
"databases/imported_database.yaml": {
"database_name": ["Missing data for required field."],
}
}
def test_import_v1_database_masked_password(self):
"""Test that database imports with masked passwords are rejected"""
masked_database_config = database_config.copy()
masked_database_config[
"sqlalchemy_uri"
] = "postgresql://*****:*****@host:12345/db"
contents = {
"metadata.yaml": yaml.safe_dump(database_metadata_config),
"databases/imported_database.yaml": yaml.safe_dump(masked_database_config),
}
command = ImportDatabasesCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing database"
assert excinfo.value.normalized_messages() == {
"databases/imported_database.yaml": {
"_schema": ["Must provide a password for the database"]
}
}
def test_import_v1_database_multiple(self):
"""Test that a database can be imported multiple times"""
num_databases = db.session.query(Database).count()
contents = {
"databases/imported_database.yaml": yaml.safe_dump(database_config),
"metadata.yaml": yaml.safe_dump(database_metadata_config),
}
command = ImportDatabasesCommand(contents, overwrite=True)
# import twice
command.run()
command.run()
database = (
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
)
assert database.allow_file_upload
# update allow_file_upload to False
new_config = database_config.copy()
new_config["allow_csv_upload"] = False
contents = {
"databases/imported_database.yaml": yaml.safe_dump(new_config),
"metadata.yaml": yaml.safe_dump(database_metadata_config),
}
command = ImportDatabasesCommand(contents, overwrite=True)
command.run()
database = (
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
)
assert not database.allow_file_upload
# test that only one database was created
new_num_databases = db.session.query(Database).count()
assert new_num_databases == num_databases + 1
db.session.delete(database)
db.session.commit()
