def validate(self) -> None:
exceptions = list()
owner_ids: Optional[List[int]] = self._properties.get("owners")
# Validate/populate model exists
self._model = DatasetDAO.find_by_id(self._model_id)
if not self._model:
raise DatasetNotFoundError()
# Check ownership
try:
check_ownership(self._model)
except SupersetSecurityException:
raise DatasetForbiddenError()
database_id = self._properties.get("database", None)
table_name = self._properties.get("table_name", None)
# Validate uniqueness
if not DatasetDAO.validate_update_uniqueness(
self._model.database_id, self._model_id, table_name):
exceptions.append(DatasetExistsValidationError(table_name))
# Validate/Populate database not allowed to change
if database_id and database_id != self._model:
exceptions.append(DatabaseChangeValidationError())
# Validate/Populate owner
try:
owners = populate_owners(self._actor, owner_ids)
self._properties["owners"] = owners
except ValidationError as e:
exceptions.append(e)
if exceptions:
exception = DatasetInvalidError()
exception.add_list(exceptions)
raise exception
def validate(self) -> None:
exceptions: List[ValidationError] = list()
database_id = self._properties["database"]
table_name = self._properties["table_name"]
schema = self._properties.get("schema", None)
owner_ids: Optional[List[int]] = self._properties.get("owners")
# Validate uniqueness
if not DatasetDAO.validate_uniqueness(database_id, schema, table_name):
exceptions.append(DatasetExistsValidationError(table_name))
# Validate/Populate database
database = DatasetDAO.get_database_by_id(database_id)
if not database:
exceptions.append(DatabaseNotFoundValidationError())
self._properties["database"] = database
# Validate table exists on dataset
if database and not DatasetDAO.validate_table_exists(
database, table_name, schema):
exceptions.append(TableNotFoundValidationError(table_name))
try:
owners = populate_owners(self._actor, owner_ids)
self._properties["owners"] = owners
except ValidationError as ex:
exceptions.append(ex)
if exceptions:
exception = DatasetInvalidError()
exception.add_list(exceptions)
raise exception
def run(self) -> None:
self.validate()
if not self._models:
return None
try:
DatasetDAO.bulk_delete(self._models)
for model in self._models:
view_menu = (security_manager.find_view_menu(model.get_perm())
if model else None)
if view_menu:
permission_views = (db.session.query(
security_manager.permissionview_model).filter_by(
view_menu=view_menu).all())
for permission_view in permission_views:
db.session.delete(permission_view)
if view_menu:
db.session.delete(view_menu)
else:
if not view_menu:
logger.error(
"Could not find the data access permission for the dataset",
exc_info=True,
)
db.session.commit()
return None
except DeleteFailedError as ex:
logger.exception(ex.exception)
raise DatasetBulkDeleteFailedError() from ex
def test_import_csv_explore_database(self):
if utils.get_example_database().backend == "sqlite":
# sqlite doesn't support schema / database creation
return
explore_db_id = utils.get_example_database().id
upload_db = utils.get_or_create_db(
"csv_explore_db", app.config["SQLALCHEMY_DATABASE_URI"])
upload_db_id = upload_db.id
extra = upload_db.get_extra()
extra["explore_database_id"] = explore_db_id
upload_db.extra = json.dumps(extra)
db.session.commit()
self.login(username="******")
self.enable_csv_upload(DatasetDAO.get_database_by_id(upload_db_id))
table_name = "".join(
random.choice(string.ascii_uppercase) for _ in range(5))
f = "testCSV.csv"
self.create_sample_csvfile(f, ["a,b", "john,1", "paul,2"])
# initial upload with fail mode
resp = self.upload_csv(f, table_name)
self.assertIn(f'CSV file "{f}" uploaded to table "{table_name}"', resp)
table = self.get_table_by_name(table_name)
self.assertEqual(table.database_id, explore_db_id)
# cleanup
db.session.delete(table)
db.session.delete(DatasetDAO.get_database_by_id(upload_db_id))
db.session.commit()
os.remove(f)
def related_objects(self, pk: int) -> Response:
"""Get charts and dashboards count associated to a dataset
---
get:
description:
Get charts and dashboards count associated to a dataset
parameters:
- in: path
name: pk
schema:
type: integer
responses:
200:
200:
description: Query result
content:
application/json:
schema:
$ref: "#/components/schemas/DatasetRelatedObjectsResponse"
401:
$ref: '#/components/responses/401'
404:
$ref: '#/components/responses/404'
500:
$ref: '#/components/responses/500'
"""
dataset = DatasetDAO.find_by_id(pk)
if not dataset:
return self.response_404()
data = DatasetDAO.get_related_objects(pk)
charts = [{
"id": chart.id,
"slice_name": chart.slice_name,
"viz_type": chart.viz_type,
} for chart in data["charts"]]
dashboards = [{
"id": dashboard.id,
"json_metadata": dashboard.json_metadata,
"slug": dashboard.slug,
"title": dashboard.dashboard_title,
} for dashboard in data["dashboards"]]
return self.response(
200,
charts={
"count": len(charts),
"result": charts
},
dashboards={
"count": len(dashboards),
"result": dashboards
},
)
def dataset_macro(
dataset_id: int,
include_metrics: bool = False,
columns: Optional[List[str]] = None,
) -> str:
"""
Given a dataset ID, return the SQL that represents it.
The generated SQL includes all columns (including computed) by default. Optionally
the user can also request metrics to be included, and columns to group by.
"""
# pylint: disable=import-outside-toplevel
from superset.datasets.dao import DatasetDAO
dataset = DatasetDAO.find_by_id(dataset_id)
if not dataset:
raise DatasetNotFoundError(f"Dataset {dataset_id} not found!")
columns = columns or [column.column_name for column in dataset.columns]
metrics = [metric.metric_name for metric in dataset.metrics]
query_obj = {
"is_timeseries": False,
"filter": [],
"metrics": metrics if include_metrics else None,
"columns": columns,
}
sqla_query = dataset.get_query_str_extended(query_obj)
sql = sqla_query.sql
return f"({sql}) AS dataset_{dataset_id}"
def run(self) -> Model:
self.validate()
try:
dataset = DatasetDAO.delete(self._model, commit=False)
view_menu = (security_manager.find_view_menu(
self._model.get_perm()) if self._model else None)
if view_menu:
permission_views = (db.session.query(
security_manager.permissionview_model).filter_by(
view_menu=view_menu).all())
for permission_view in permission_views:
db.session.delete(permission_view)
if view_menu:
db.session.delete(view_menu)
else:
if not view_menu:
logger.error(
"Could not find the data access permission for the dataset"
)
db.session.commit()
except (SQLAlchemyError, DAODeleteFailedError) as ex:
logger.exception(ex)
db.session.rollback()
raise DatasetDeleteFailedError()
return dataset
def _export(
model: Dashboard, export_related: bool = True
) -> Iterator[Tuple[str, str]]:
dashboard_slug = secure_filename(model.dashboard_title)
file_name = f"dashboards/{dashboard_slug}.yaml"
payload = model.export_to_dict(
recursive=False,
include_parent_ref=False,
include_defaults=True,
export_uuids=True,
)
# TODO (betodealmeida): move this logic to export_to_dict once this
# becomes the default export endpoint
for key, new_name in JSON_KEYS.items():
value: Optional[str] = payload.pop(key, None)
if value:
try:
payload[new_name] = json.loads(value)
except (TypeError, json.decoder.JSONDecodeError):
logger.info("Unable to decode `%s` field: %s", key, value)
payload[new_name] = {}
# Extract all native filter datasets and replace native
# filter dataset references with uuid
for native_filter in payload.get("metadata", {}).get(
"native_filter_configuration", []
):
for target in native_filter.get("targets", []):
dataset_id = target.pop("datasetId", None)
if dataset_id is not None:
dataset = DatasetDAO.find_by_id(dataset_id)
target["datasetUuid"] = str(dataset.uuid)
if export_related:
yield from ExportDatasetsCommand([dataset_id]).run()
# the mapping between dashboard -> charts is inferred from the position
# attribute, so if it's not present we need to add a default config
if not payload.get("position"):
payload["position"] = get_default_position(model.dashboard_title)
# if any charts or not referenced in position, we need to add them
# in a new row
referenced_charts = find_chart_uuids(payload["position"])
orphan_charts = {
chart for chart in model.slices if str(chart.uuid) not in referenced_charts
}
if orphan_charts:
payload["position"] = append_charts(payload["position"], orphan_charts)
payload["version"] = EXPORT_VERSION
file_content = yaml.safe_dump(payload, sort_keys=False)
yield file_name, file_content
if export_related:
chart_ids = [chart.id for chart in model.slices]
yield from ExportChartsCommand(chart_ids).run()
def run(self):
self.validate()
try:
dataset = DatasetDAO.update(self._model, self._properties)
except UpdateFailedError as e:
logger.exception(e.exception)
raise DatasetUpdateFailedError()
return dataset
def run(self):
self.validate()
try:
dataset = DatasetDAO.delete(self._model)
except DeleteFailedError as e:
logger.exception(e.exception)
raise DatasetDeleteFailedError()
return dataset
def check_dataset_access(dataset_id: int) -> Optional[bool]:
if dataset_id:
dataset = DatasetDAO.find_by_id(dataset_id)
if dataset:
can_access_datasource = security_manager.can_access_datasource(dataset)
if can_access_datasource:
return True
raise DatasetAccessDeniedError()
raise DatasetNotFoundError()
def validate(self) -> None:
exceptions: List[ValidationError] = []
owner_ids: Optional[List[int]] = self._properties.get("owners")
# Validate/populate model exists
self._model = DatasetDAO.find_by_id(self._model_id)
if not self._model:
raise DatasetNotFoundError()
# Check ownership
try:
security_manager.raise_for_ownership(self._model)
except SupersetSecurityException as ex:
raise DatasetForbiddenError() from ex
database_id = self._properties.get("database", None)
table_name = self._properties.get("table_name", None)
# Validate uniqueness
if not DatasetDAO.validate_update_uniqueness(
self._model.database_id, self._model_id, table_name):
exceptions.append(DatasetExistsValidationError(table_name))
# Validate/Populate database not allowed to change
if database_id and database_id != self._model:
exceptions.append(DatabaseChangeValidationError())
# Validate/Populate owner
try:
owners = self.populate_owners(owner_ids)
self._properties["owners"] = owners
except ValidationError as ex:
exceptions.append(ex)
# Validate columns
columns = self._properties.get("columns")
if columns:
self._validate_columns(columns, exceptions)
# Validate metrics
metrics = self._properties.get("metrics")
if metrics:
self._validate_metrics(metrics, exceptions)
if exceptions:
exception = DatasetInvalidError()
exception.add_list(exceptions)
raise exception
def validate(self) -> None:
# Validate/populate model exists
self._model = DatasetDAO.find_by_id(self._model_id)
if not self._model:
raise DatasetNotFoundError()
# Check ownership
try:
check_ownership(self._model)
except SupersetSecurityException:
raise DatasetForbiddenError()
def check_dataset_access(dataset_id: int) -> Optional[bool]:
if dataset_id:
# Access checks below, no need to validate them twice as they can be expensive.
dataset = DatasetDAO.find_by_id(dataset_id, skip_base_filter=True)
if dataset:
can_access_datasource = security_manager.can_access_datasource(dataset)
if can_access_datasource:
return True
raise DatasetAccessDeniedError()
raise DatasetNotFoundError()
def run(self) -> Model:
self.validate()
try:
if not self._model:
raise DatasetMetricNotFoundError()
column = DatasetDAO.delete_metric(self._model)
return column
except DAODeleteFailedError as ex:
logger.exception(ex.exception)
raise DatasetMetricDeleteFailedError() from ex
def run(self) -> Model:
self.validate()
if self._model:
try:
dataset = DatasetDAO.update(self._model, self._properties)
return dataset
except DAOUpdateFailedError as ex:
logger.exception(ex.exception)
raise DatasetUpdateFailedError()
raise DatasetUpdateFailedError()
def test_datasource_find_by_id_skip_base_filter_not_found(
session_with_data: Session, ) -> None:
from superset.datasets.dao import DatasetDAO
result = DatasetDAO.find_by_id(
125326326,
session=session_with_data,
skip_base_filter=True,
)
assert result is None
def validate(self) -> None:
# Validate/populate model exists
self._model = DatasetDAO.find_dataset_metric(self._dataset_id, self._model_id)
if not self._model:
raise DatasetMetricNotFoundError()
# Check ownership
try:
check_ownership(self._model)
except SupersetSecurityException as ex:
raise DatasetMetricForbiddenError() from ex
def validate(self) -> None:
# Validate/populate model exists
self._models = DatasetDAO.find_by_ids(self._model_ids)
if not self._models or len(self._models) != len(self._model_ids):
raise DatasetNotFoundError()
# Check ownership
for model in self._models:
try:
security_manager.raise_for_ownership(model)
except SupersetSecurityException as ex:
raise DatasetForbiddenError() from ex
def _validate_metrics(self, metrics: List[Dict],
exceptions: List[ValidationError]) -> None:
if self._get_duplicates(metrics, "metric_name"):
exceptions.append(DatasetMetricsDuplicateValidationError())
else:
# validate invalid id's
metrics_ids: List[int] = [
metric["id"] for metric in metrics if "id" in metric
]
if not DatasetDAO.validate_metrics_exist(self._model_id,
metrics_ids):
exceptions.append(DatasetMetricsNotFoundValidationError())
# validate new metric names uniqueness
metric_names: List[str] = [
metric["metric_name"] for metric in metrics
if "id" not in metric
]
if not DatasetDAO.validate_metrics_uniqueness(
self._model_id, metric_names):
exceptions.append(DatasetMetricsExistsValidationError())
def validate(self) -> None:
# Validate/populate model exists
self._model = DatasetDAO.find_dataset_column(self._dataset_id,
self._model_id)
if not self._model:
raise DatasetColumnNotFoundError()
# Check ownership
try:
security_manager.raise_for_ownership(self._model)
except SupersetSecurityException as ex:
raise DatasetColumnForbiddenError() from ex
def _validate_columns(self, columns: List[Dict],
exceptions: List[ValidationError]) -> None:
# Validate duplicates on data
if self._get_duplicates(columns, "column_name"):
exceptions.append(DatasetColumnsDuplicateValidationError())
else:
# validate invalid id's
columns_ids: List[int] = [
column["id"] for column in columns if "id" in column
]
if not DatasetDAO.validate_columns_exist(self._model_id,
columns_ids):
exceptions.append(DatasetColumnNotFoundValidationError())
# validate new column names uniqueness
columns_names: List[str] = [
column["column_name"] for column in columns
if "id" not in column
]
if not DatasetDAO.validate_columns_uniqueness(
self._model_id, columns_names):
exceptions.append(DatasetColumnsExistsValidationError())
def run(self) -> Model:
self.validate()
try:
dataset = DatasetDAO.delete(self._model, commit=False)
security_manager.del_permission_view_menu("datasource_access",
dataset.get_perm())
db.session.commit()
except (SQLAlchemyError, DAODeleteFailedError) as e:
logger.exception(e)
db.session.rollback()
raise DatasetDeleteFailedError()
return dataset
def run(self) -> Model:
self.validate()
if self._model:
try:
dataset = DatasetDAO.update(
model=self._model,
properties=self._properties,
override_columns=self.override_columns,
)
return dataset
except DAOUpdateFailedError as ex:
logger.exception(ex.exception)
raise DatasetUpdateFailedError()
raise DatasetUpdateFailedError()
def test_datasource_find_by_id_skip_base_filter(
session_with_data: Session) -> None:
from superset.connectors.sqla.models import SqlaTable
from superset.datasets.dao import DatasetDAO
result = DatasetDAO.find_by_id(
1,
session=session_with_data,
skip_base_filter=True,
)
assert result
assert 1 == result.id
assert "my_sqla_table" == result.table_name
assert isinstance(result, SqlaTable)
def run(self) -> Model:
self.validate()
try:
# Creates SqlaTable (Dataset)
dataset = DatasetDAO.create(self._properties, commit=False)
# Updates columns and metrics from the dataset
dataset.fetch_metadata(commit=False)
# Add datasource access permission
security_manager.add_permission_view_menu("datasource_access",
dataset.get_perm())
# Add schema access permission if exists
if dataset.schema:
security_manager.add_permission_view_menu(
"schema_access", dataset.schema_perm)
db.session.commit()
except (SQLAlchemyError, DAOCreateFailedError) as ex:
logger.warning(ex, exc_info=True)
db.session.rollback()
raise DatasetCreateFailedError()
return dataset
def populate_owners(user: User,
owners_ids: Optional[List[int]] = None) -> List[User]:
"""
Helper function for commands, will fetch all users from owners id's
Can raise ValidationError
:param user: The current user
:param owners_ids: A List of owners by id's
"""
owners = list()
if not owners_ids:
return [user]
if user.id not in owners_ids:
owners.append(user)
for owner_id in owners_ids:
owner = DatasetDAO.get_owner_by_id(owner_id)
if not owner:
raise OwnersNotFoundValidationError()
owners.append(owner)
return owners
def validate(self) -> None:
self._models = DatasetDAO.find_by_ids(self.dataset_ids)
if len(self._models) != len(self.dataset_ids):
raise DatasetNotFoundError()
