def test_import_v1_chart_validation(self):
"""Test different validations applied when importing a chart"""
# metadata.yaml must be present
contents = {
"databases/imported_database.yaml":
yaml.safe_dump(database_config),
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
"charts/imported_chart.yaml": yaml.safe_dump(chart_config),
}
command = ImportChartsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Missing metadata.yaml"
# version should be 1.0.0
contents["metadata.yaml"] = yaml.safe_dump({
"version":
"2.0.0",
"type":
"SqlaTable",
"timestamp":
"2020-11-04T21:27:44.423819+00:00",
})
command = ImportChartsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Must be equal to 1.0.0."
# type should be Slice
contents["metadata.yaml"] = yaml.safe_dump(database_metadata_config)
command = ImportChartsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing chart"
assert excinfo.value.normalized_messages() == {
"metadata.yaml": {
"type": ["Must be equal to Slice."]
}
}
# must also validate datasets and databases
broken_config = database_config.copy()
del broken_config["database_name"]
contents["metadata.yaml"] = yaml.safe_dump(chart_metadata_config)
contents["databases/imported_database.yaml"] = yaml.safe_dump(
broken_config)
command = ImportChartsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing chart"
assert excinfo.value.normalized_messages() == {
"databases/imported_database.yaml": {
"database_name": ["Missing data for required field."],
}
}
def test_import_v1_saved_queries_validation(self):
"""Test different validations applied when importing a saved query"""
# metadata.yaml must be present
contents = {
"databases/imported_database.yaml": yaml.safe_dump(database_config),
"queries/imported_query.yaml": yaml.safe_dump(saved_queries_config),
}
command = ImportSavedQueriesCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Missing metadata.yaml"
# version should be 1.0.0
contents["metadata.yaml"] = yaml.safe_dump(
{
"version": "2.0.0",
"type": "SavedQuery",
"timestamp": "2021-03-30T20:37:54.791187+00:00",
}
)
command = ImportSavedQueriesCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Must be equal to 1.0.0."
# type should be a SavedQuery
contents["metadata.yaml"] = yaml.safe_dump(database_metadata_config)
command = ImportSavedQueriesCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing saved_queries"
assert excinfo.value.normalized_messages() == {
"metadata.yaml": {"type": ["Must be equal to SavedQuery."]}
}
# must also validate databases
broken_config = database_config.copy()
del broken_config["database_name"]
contents["metadata.yaml"] = yaml.safe_dump(saved_queries_metadata_config)
contents["databases/imported_database.yaml"] = yaml.safe_dump(broken_config)
command = ImportSavedQueriesCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing saved_queries"
assert excinfo.value.normalized_messages() == {
"databases/imported_database.yaml": {
"database_name": ["Missing data for required field."],
}
}
def test_import_database_masked_password_provided(self):
"""
Database API: Test import database with masked password provided
"""
self.login(username="******")
uri = "api/v1/database/import/"
masked_database_config = database_config.copy()
masked_database_config[
"sqlalchemy_uri"] = "postgresql://*****:*****@host:12345/db"
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("database_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(database_metadata_config).encode())
with bundle.open(
"database_export/databases/imported_database.yaml",
"w") as fp:
fp.write(yaml.safe_dump(masked_database_config).encode())
buf.seek(0)
form_data = {
"formData": (buf, "database_export.zip"),
"passwords":
json.dumps({"databases/imported_database.yaml": "SECRET"}),
}
rv = self.client.post(uri,
data=form_data,
content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))
assert rv.status_code == 200
assert response == {"message": "OK"}
database = (db.session.query(Database).filter_by(
uuid=database_config["uuid"]).one())
assert database.database_name == "imported_database"
assert (database.sqlalchemy_uri ==
"postgresql://*****:*****@host:12345/db")
assert database.password == "SECRET"
db.session.delete(database)
db.session.commit()
def test_import_v1_database_masked_password(self):
"""Test that database imports with masked passwords are rejected"""
masked_database_config = database_config.copy()
masked_database_config[
"sqlalchemy_uri"
] = "postgresql://*****:*****@host:12345/db"
contents = {
"metadata.yaml": yaml.safe_dump(database_metadata_config),
"databases/imported_database.yaml": yaml.safe_dump(masked_database_config),
}
command = ImportDatabasesCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing database"
assert excinfo.value.normalized_messages() == {
"databases/imported_database.yaml": {
"_schema": ["Must provide a password for the database"]
}
}
def test_import_database_masked_password(self):
"""
Database API: Test import database with masked password
"""
self.login(username="******")
uri = "api/v1/database/import/"
masked_database_config = database_config.copy()
masked_database_config[
"sqlalchemy_uri"] = "postgresql://*****:*****@host:12345/db"
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
with bundle.open("database_export/metadata.yaml", "w") as fp:
fp.write(yaml.safe_dump(database_metadata_config).encode())
with bundle.open(
"database_export/databases/imported_database.yaml",
"w") as fp:
fp.write(yaml.safe_dump(masked_database_config).encode())
with bundle.open("database_export/datasets/imported_dataset.yaml",
"w") as fp:
fp.write(yaml.safe_dump(dataset_config).encode())
buf.seek(0)
form_data = {
"formData": (buf, "database_export.zip"),
}
rv = self.client.post(uri,
data=form_data,
content_type="multipart/form-data")
response = json.loads(rv.data.decode("utf-8"))
assert rv.status_code == 422
assert response == {
"message": {
"databases/imported_database.yaml": {
"_schema": ["Must provide a password for the database"]
}
}
}
def test_import_v1_database_multiple(self):
"""Test that a database can be imported multiple times"""
num_databases = db.session.query(Database).count()
contents = {
"databases/imported_database.yaml": yaml.safe_dump(database_config),
"metadata.yaml": yaml.safe_dump(database_metadata_config),
}
command = ImportDatabasesCommand(contents)
# import twice
command.run()
command.run()
database = (
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
)
assert database.allow_csv_upload
# update allow_csv_upload to False
new_config = database_config.copy()
new_config["allow_csv_upload"] = False
contents = {
"databases/imported_database.yaml": yaml.safe_dump(new_config),
"metadata.yaml": yaml.safe_dump(database_metadata_config),
}
command = ImportDatabasesCommand(contents)
command.run()
database = (
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
)
assert not database.allow_csv_upload
# test that only one database was created
new_num_databases = db.session.query(Database).count()
assert new_num_databases == num_databases + 1
db.session.delete(database)
db.session.commit()
