def validate(self) -> None: exceptions = list() owner_ids: Optional[List[int]] = self._properties.get("owners") # Validate/populate model exists self._model = DatasetDAO.find_by_id(self._model_id) if not self._model: raise DatasetNotFoundError() # Check ownership try: check_ownership(self._model) except SupersetSecurityException: raise DatasetForbiddenError() database_id = self._properties.get("database", None) table_name = self._properties.get("table_name", None) # Validate uniqueness if not DatasetDAO.validate_update_uniqueness( self._model.database_id, self._model_id, table_name): exceptions.append(DatasetExistsValidationError(table_name)) # Validate/Populate database not allowed to change if database_id and database_id != self._model: exceptions.append(DatabaseChangeValidationError()) # Validate/Populate owner try: owners = populate_owners(self._actor, owner_ids) self._properties["owners"] = owners except ValidationError as e: exceptions.append(e) if exceptions: exception = DatasetInvalidError() exception.add_list(exceptions) raise exception
def validate(self) -> None: exceptions: List[ValidationError] = list() database_id = self._properties["database"] table_name = self._properties["table_name"] schema = self._properties.get("schema", None) owner_ids: Optional[List[int]] = self._properties.get("owners") # Validate uniqueness if not DatasetDAO.validate_uniqueness(database_id, schema, table_name): exceptions.append(DatasetExistsValidationError(table_name)) # Validate/Populate database database = DatasetDAO.get_database_by_id(database_id) if not database: exceptions.append(DatabaseNotFoundValidationError()) self._properties["database"] = database # Validate table exists on dataset if database and not DatasetDAO.validate_table_exists( database, table_name, schema): exceptions.append(TableNotFoundValidationError(table_name)) try: owners = populate_owners(self._actor, owner_ids) self._properties["owners"] = owners except ValidationError as ex: exceptions.append(ex) if exceptions: exception = DatasetInvalidError() exception.add_list(exceptions) raise exception
def run(self) -> None: self.validate() if not self._models: return None try: DatasetDAO.bulk_delete(self._models) for model in self._models: view_menu = (security_manager.find_view_menu(model.get_perm()) if model else None) if view_menu: permission_views = (db.session.query( security_manager.permissionview_model).filter_by( view_menu=view_menu).all()) for permission_view in permission_views: db.session.delete(permission_view) if view_menu: db.session.delete(view_menu) else: if not view_menu: logger.error( "Could not find the data access permission for the dataset", exc_info=True, ) db.session.commit() return None except DeleteFailedError as ex: logger.exception(ex.exception) raise DatasetBulkDeleteFailedError() from ex
def test_import_csv_explore_database(self): if utils.get_example_database().backend == "sqlite": # sqlite doesn't support schema / database creation return explore_db_id = utils.get_example_database().id upload_db = utils.get_or_create_db( "csv_explore_db", app.config["SQLALCHEMY_DATABASE_URI"]) upload_db_id = upload_db.id extra = upload_db.get_extra() extra["explore_database_id"] = explore_db_id upload_db.extra = json.dumps(extra) db.session.commit() self.login(username="******") self.enable_csv_upload(DatasetDAO.get_database_by_id(upload_db_id)) table_name = "".join( random.choice(string.ascii_uppercase) for _ in range(5)) f = "testCSV.csv" self.create_sample_csvfile(f, ["a,b", "john,1", "paul,2"]) # initial upload with fail mode resp = self.upload_csv(f, table_name) self.assertIn(f'CSV file "{f}" uploaded to table "{table_name}"', resp) table = self.get_table_by_name(table_name) self.assertEqual(table.database_id, explore_db_id) # cleanup db.session.delete(table) db.session.delete(DatasetDAO.get_database_by_id(upload_db_id)) db.session.commit() os.remove(f)
def related_objects(self, pk: int) -> Response: """Get charts and dashboards count associated to a dataset --- get: description: Get charts and dashboards count associated to a dataset parameters: - in: path name: pk schema: type: integer responses: 200: 200: description: Query result content: application/json: schema: $ref: "#/components/schemas/DatasetRelatedObjectsResponse" 401: $ref: '#/components/responses/401' 404: $ref: '#/components/responses/404' 500: $ref: '#/components/responses/500' """ dataset = DatasetDAO.find_by_id(pk) if not dataset: return self.response_404() data = DatasetDAO.get_related_objects(pk) charts = [{ "id": chart.id, "slice_name": chart.slice_name, "viz_type": chart.viz_type, } for chart in data["charts"]] dashboards = [{ "id": dashboard.id, "json_metadata": dashboard.json_metadata, "slug": dashboard.slug, "title": dashboard.dashboard_title, } for dashboard in data["dashboards"]] return self.response( 200, charts={ "count": len(charts), "result": charts }, dashboards={ "count": len(dashboards), "result": dashboards }, )
def dataset_macro( dataset_id: int, include_metrics: bool = False, columns: Optional[List[str]] = None, ) -> str: """ Given a dataset ID, return the SQL that represents it. The generated SQL includes all columns (including computed) by default. Optionally the user can also request metrics to be included, and columns to group by. """ # pylint: disable=import-outside-toplevel from superset.datasets.dao import DatasetDAO dataset = DatasetDAO.find_by_id(dataset_id) if not dataset: raise DatasetNotFoundError(f"Dataset {dataset_id} not found!") columns = columns or [column.column_name for column in dataset.columns] metrics = [metric.metric_name for metric in dataset.metrics] query_obj = { "is_timeseries": False, "filter": [], "metrics": metrics if include_metrics else None, "columns": columns, } sqla_query = dataset.get_query_str_extended(query_obj) sql = sqla_query.sql return f"({sql}) AS dataset_{dataset_id}"
def run(self) -> Model: self.validate() try: dataset = DatasetDAO.delete(self._model, commit=False) view_menu = (security_manager.find_view_menu( self._model.get_perm()) if self._model else None) if view_menu: permission_views = (db.session.query( security_manager.permissionview_model).filter_by( view_menu=view_menu).all()) for permission_view in permission_views: db.session.delete(permission_view) if view_menu: db.session.delete(view_menu) else: if not view_menu: logger.error( "Could not find the data access permission for the dataset" ) db.session.commit() except (SQLAlchemyError, DAODeleteFailedError) as ex: logger.exception(ex) db.session.rollback() raise DatasetDeleteFailedError() return dataset
def _export( model: Dashboard, export_related: bool = True ) -> Iterator[Tuple[str, str]]: dashboard_slug = secure_filename(model.dashboard_title) file_name = f"dashboards/{dashboard_slug}.yaml" payload = model.export_to_dict( recursive=False, include_parent_ref=False, include_defaults=True, export_uuids=True, ) # TODO (betodealmeida): move this logic to export_to_dict once this # becomes the default export endpoint for key, new_name in JSON_KEYS.items(): value: Optional[str] = payload.pop(key, None) if value: try: payload[new_name] = json.loads(value) except (TypeError, json.decoder.JSONDecodeError): logger.info("Unable to decode `%s` field: %s", key, value) payload[new_name] = {} # Extract all native filter datasets and replace native # filter dataset references with uuid for native_filter in payload.get("metadata", {}).get( "native_filter_configuration", [] ): for target in native_filter.get("targets", []): dataset_id = target.pop("datasetId", None) if dataset_id is not None: dataset = DatasetDAO.find_by_id(dataset_id) target["datasetUuid"] = str(dataset.uuid) if export_related: yield from ExportDatasetsCommand([dataset_id]).run() # the mapping between dashboard -> charts is inferred from the position # attribute, so if it's not present we need to add a default config if not payload.get("position"): payload["position"] = get_default_position(model.dashboard_title) # if any charts or not referenced in position, we need to add them # in a new row referenced_charts = find_chart_uuids(payload["position"]) orphan_charts = { chart for chart in model.slices if str(chart.uuid) not in referenced_charts } if orphan_charts: payload["position"] = append_charts(payload["position"], orphan_charts) payload["version"] = EXPORT_VERSION file_content = yaml.safe_dump(payload, sort_keys=False) yield file_name, file_content if export_related: chart_ids = [chart.id for chart in model.slices] yield from ExportChartsCommand(chart_ids).run()
def run(self): self.validate() try: dataset = DatasetDAO.update(self._model, self._properties) except UpdateFailedError as e: logger.exception(e.exception) raise DatasetUpdateFailedError() return dataset
def run(self): self.validate() try: dataset = DatasetDAO.delete(self._model) except DeleteFailedError as e: logger.exception(e.exception) raise DatasetDeleteFailedError() return dataset
def check_dataset_access(dataset_id: int) -> Optional[bool]: if dataset_id: dataset = DatasetDAO.find_by_id(dataset_id) if dataset: can_access_datasource = security_manager.can_access_datasource(dataset) if can_access_datasource: return True raise DatasetAccessDeniedError() raise DatasetNotFoundError()
def validate(self) -> None: exceptions: List[ValidationError] = [] owner_ids: Optional[List[int]] = self._properties.get("owners") # Validate/populate model exists self._model = DatasetDAO.find_by_id(self._model_id) if not self._model: raise DatasetNotFoundError() # Check ownership try: security_manager.raise_for_ownership(self._model) except SupersetSecurityException as ex: raise DatasetForbiddenError() from ex database_id = self._properties.get("database", None) table_name = self._properties.get("table_name", None) # Validate uniqueness if not DatasetDAO.validate_update_uniqueness( self._model.database_id, self._model_id, table_name): exceptions.append(DatasetExistsValidationError(table_name)) # Validate/Populate database not allowed to change if database_id and database_id != self._model: exceptions.append(DatabaseChangeValidationError()) # Validate/Populate owner try: owners = self.populate_owners(owner_ids) self._properties["owners"] = owners except ValidationError as ex: exceptions.append(ex) # Validate columns columns = self._properties.get("columns") if columns: self._validate_columns(columns, exceptions) # Validate metrics metrics = self._properties.get("metrics") if metrics: self._validate_metrics(metrics, exceptions) if exceptions: exception = DatasetInvalidError() exception.add_list(exceptions) raise exception
def validate(self) -> None: # Validate/populate model exists self._model = DatasetDAO.find_by_id(self._model_id) if not self._model: raise DatasetNotFoundError() # Check ownership try: check_ownership(self._model) except SupersetSecurityException: raise DatasetForbiddenError()
def check_dataset_access(dataset_id: int) -> Optional[bool]: if dataset_id: # Access checks below, no need to validate them twice as they can be expensive. dataset = DatasetDAO.find_by_id(dataset_id, skip_base_filter=True) if dataset: can_access_datasource = security_manager.can_access_datasource(dataset) if can_access_datasource: return True raise DatasetAccessDeniedError() raise DatasetNotFoundError()
def run(self) -> Model: self.validate() try: if not self._model: raise DatasetMetricNotFoundError() column = DatasetDAO.delete_metric(self._model) return column except DAODeleteFailedError as ex: logger.exception(ex.exception) raise DatasetMetricDeleteFailedError() from ex
def run(self) -> Model: self.validate() if self._model: try: dataset = DatasetDAO.update(self._model, self._properties) return dataset except DAOUpdateFailedError as ex: logger.exception(ex.exception) raise DatasetUpdateFailedError() raise DatasetUpdateFailedError()
def test_datasource_find_by_id_skip_base_filter_not_found( session_with_data: Session, ) -> None: from superset.datasets.dao import DatasetDAO result = DatasetDAO.find_by_id( 125326326, session=session_with_data, skip_base_filter=True, ) assert result is None
def validate(self) -> None: # Validate/populate model exists self._model = DatasetDAO.find_dataset_metric(self._dataset_id, self._model_id) if not self._model: raise DatasetMetricNotFoundError() # Check ownership try: check_ownership(self._model) except SupersetSecurityException as ex: raise DatasetMetricForbiddenError() from ex
def validate(self) -> None: # Validate/populate model exists self._models = DatasetDAO.find_by_ids(self._model_ids) if not self._models or len(self._models) != len(self._model_ids): raise DatasetNotFoundError() # Check ownership for model in self._models: try: security_manager.raise_for_ownership(model) except SupersetSecurityException as ex: raise DatasetForbiddenError() from ex
def _validate_metrics(self, metrics: List[Dict], exceptions: List[ValidationError]) -> None: if self._get_duplicates(metrics, "metric_name"): exceptions.append(DatasetMetricsDuplicateValidationError()) else: # validate invalid id's metrics_ids: List[int] = [ metric["id"] for metric in metrics if "id" in metric ] if not DatasetDAO.validate_metrics_exist(self._model_id, metrics_ids): exceptions.append(DatasetMetricsNotFoundValidationError()) # validate new metric names uniqueness metric_names: List[str] = [ metric["metric_name"] for metric in metrics if "id" not in metric ] if not DatasetDAO.validate_metrics_uniqueness( self._model_id, metric_names): exceptions.append(DatasetMetricsExistsValidationError())
def validate(self) -> None: # Validate/populate model exists self._model = DatasetDAO.find_dataset_column(self._dataset_id, self._model_id) if not self._model: raise DatasetColumnNotFoundError() # Check ownership try: security_manager.raise_for_ownership(self._model) except SupersetSecurityException as ex: raise DatasetColumnForbiddenError() from ex
def _validate_columns(self, columns: List[Dict], exceptions: List[ValidationError]) -> None: # Validate duplicates on data if self._get_duplicates(columns, "column_name"): exceptions.append(DatasetColumnsDuplicateValidationError()) else: # validate invalid id's columns_ids: List[int] = [ column["id"] for column in columns if "id" in column ] if not DatasetDAO.validate_columns_exist(self._model_id, columns_ids): exceptions.append(DatasetColumnNotFoundValidationError()) # validate new column names uniqueness columns_names: List[str] = [ column["column_name"] for column in columns if "id" not in column ] if not DatasetDAO.validate_columns_uniqueness( self._model_id, columns_names): exceptions.append(DatasetColumnsExistsValidationError())
def run(self) -> Model: self.validate() try: dataset = DatasetDAO.delete(self._model, commit=False) security_manager.del_permission_view_menu("datasource_access", dataset.get_perm()) db.session.commit() except (SQLAlchemyError, DAODeleteFailedError) as e: logger.exception(e) db.session.rollback() raise DatasetDeleteFailedError() return dataset
def run(self) -> Model: self.validate() if self._model: try: dataset = DatasetDAO.update( model=self._model, properties=self._properties, override_columns=self.override_columns, ) return dataset except DAOUpdateFailedError as ex: logger.exception(ex.exception) raise DatasetUpdateFailedError() raise DatasetUpdateFailedError()
def test_datasource_find_by_id_skip_base_filter( session_with_data: Session) -> None: from superset.connectors.sqla.models import SqlaTable from superset.datasets.dao import DatasetDAO result = DatasetDAO.find_by_id( 1, session=session_with_data, skip_base_filter=True, ) assert result assert 1 == result.id assert "my_sqla_table" == result.table_name assert isinstance(result, SqlaTable)
def run(self) -> Model: self.validate() try: # Creates SqlaTable (Dataset) dataset = DatasetDAO.create(self._properties, commit=False) # Updates columns and metrics from the dataset dataset.fetch_metadata(commit=False) # Add datasource access permission security_manager.add_permission_view_menu("datasource_access", dataset.get_perm()) # Add schema access permission if exists if dataset.schema: security_manager.add_permission_view_menu( "schema_access", dataset.schema_perm) db.session.commit() except (SQLAlchemyError, DAOCreateFailedError) as ex: logger.warning(ex, exc_info=True) db.session.rollback() raise DatasetCreateFailedError() return dataset
def populate_owners(user: User, owners_ids: Optional[List[int]] = None) -> List[User]: """ Helper function for commands, will fetch all users from owners id's Can raise ValidationError :param user: The current user :param owners_ids: A List of owners by id's """ owners = list() if not owners_ids: return [user] if user.id not in owners_ids: owners.append(user) for owner_id in owners_ids: owner = DatasetDAO.get_owner_by_id(owner_id) if not owner: raise OwnersNotFoundValidationError() owners.append(owner) return owners
def validate(self) -> None: self._models = DatasetDAO.find_by_ids(self.dataset_ids) if len(self._models) != len(self.dataset_ids): raise DatasetNotFoundError()