def _import(session: Session, configs: Dict[str, Any]) -> None: # import databases first database_ids: Dict[str, int] = {} for file_name, config in configs.items(): if file_name.startswith("databases/"): database = import_database(session, config, overwrite=True) database_ids[str(database.uuid)] = database.id # import saved queries for file_name, config in configs.items(): if file_name.startswith("queries/"): config["db_id"] = database_ids[config["database_uuid"]] import_saved_query(session, config, overwrite=True) # import datasets dataset_info: Dict[str, Dict[str, Any]] = {} for file_name, config in configs.items(): if file_name.startswith("datasets/"): config["database_id"] = database_ids[config["database_uuid"]] dataset = import_dataset(session, config, overwrite=True) dataset_info[str(dataset.uuid)] = { "datasource_id": dataset.id, "datasource_type": dataset.datasource_type, "datasource_name": dataset.table_name, } # import charts chart_ids: Dict[str, int] = {} for file_name, config in configs.items(): if file_name.startswith("charts/"): config.update(dataset_info[config["dataset_uuid"]]) chart = import_chart(session, config, overwrite=True) chart_ids[str(chart.uuid)] = chart.id # store the existing relationship between dashboards and charts existing_relationships = session.execute( select( [dashboard_slices.c.dashboard_id, dashboard_slices.c.slice_id])).fetchall() # import dashboards dashboard_chart_ids: List[Tuple[int, int]] = [] for file_name, config in configs.items(): if file_name.startswith("dashboards/"): config = update_id_refs(config, chart_ids, dataset_info) dashboard = import_dashboard(session, config, overwrite=True) for uuid in find_chart_uuids(config["position"]): if uuid not in chart_ids: break chart_id = chart_ids[uuid] if (dashboard.id, chart_id) not in existing_relationships: dashboard_chart_ids.append((dashboard.id, chart_id)) # set ref in the dashboard_slices table values = [{ "dashboard_id": dashboard_id, "slice_id": chart_id } for (dashboard_id, chart_id) in dashboard_chart_ids] # pylint: disable=no-value-for-parameter # sqlalchemy/issues/4656 session.execute(dashboard_slices.insert(), values)
def _export( model: Dashboard, export_related: bool = True ) -> Iterator[Tuple[str, str]]: dashboard_slug = secure_filename(model.dashboard_title) file_name = f"dashboards/{dashboard_slug}.yaml" payload = model.export_to_dict( recursive=False, include_parent_ref=False, include_defaults=True, export_uuids=True, ) # TODO (betodealmeida): move this logic to export_to_dict once this # becomes the default export endpoint for key, new_name in JSON_KEYS.items(): value: Optional[str] = payload.pop(key, None) if value: try: payload[new_name] = json.loads(value) except (TypeError, json.decoder.JSONDecodeError): logger.info("Unable to decode `%s` field: %s", key, value) payload[new_name] = {} # Extract all native filter datasets and replace native # filter dataset references with uuid for native_filter in payload.get("metadata", {}).get( "native_filter_configuration", [] ): for target in native_filter.get("targets", []): dataset_id = target.pop("datasetId", None) if dataset_id is not None: dataset = DatasetDAO.find_by_id(dataset_id) target["datasetUuid"] = str(dataset.uuid) if export_related: yield from ExportDatasetsCommand([dataset_id]).run() # the mapping between dashboard -> charts is inferred from the position # attribute, so if it's not present we need to add a default config if not payload.get("position"): payload["position"] = get_default_position(model.dashboard_title) # if any charts or not referenced in position, we need to add them # in a new row referenced_charts = find_chart_uuids(payload["position"]) orphan_charts = { chart for chart in model.slices if str(chart.uuid) not in referenced_charts } if orphan_charts: payload["position"] = append_charts(payload["position"], orphan_charts) payload["version"] = EXPORT_VERSION file_content = yaml.safe_dump(payload, sort_keys=False) yield file_name, file_content if export_related: chart_ids = [chart.id for chart in model.slices] yield from ExportChartsCommand(chart_ids).run()
def _export(model: Dashboard) -> Iterator[Tuple[str, str]]: dashboard_slug = secure_filename(model.dashboard_title) file_name = f"dashboards/{dashboard_slug}.yaml" payload = model.export_to_dict( recursive=False, include_parent_ref=False, include_defaults=True, export_uuids=True, ) # TODO (betodealmeida): move this logic to export_to_dict once this # becomes the default export endpoint for key, new_name in JSON_KEYS.items(): value: Optional[str] = payload.pop(key, None) if value: try: payload[new_name] = json.loads(value) except (TypeError, json.decoder.JSONDecodeError): logger.info("Unable to decode `%s` field: %s", key, value) payload[new_name] = {} # the mapping between dashboard -> charts is inferred from the position # attribute, so if it's not present we need to add a default config if not payload.get("position"): payload["position"] = get_default_position(model.dashboard_title) # if any charts or not referenced in position, we need to add them # in a new row referenced_charts = find_chart_uuids(payload["position"]) orphan_charts = { chart for chart in model.slices if str(chart.uuid) not in referenced_charts } payload["position"] = append_charts(payload["position"], orphan_charts) payload["version"] = EXPORT_VERSION file_content = yaml.safe_dump(payload, sort_keys=False) yield file_name, file_content chart_ids = [chart.id for chart in model.slices] yield from ExportChartsCommand(chart_ids).run()
def _import(session: Session, configs: Dict[str, Any], overwrite: bool = False) -> None: # import databases database_ids: Dict[str, int] = {} for file_name, config in configs.items(): if file_name.startswith("databases/"): database = import_database(session, config, overwrite=overwrite) database_ids[str(database.uuid)] = database.id # import datasets # TODO (betodealmeida): once we have all examples being imported we can # have a stable UUID for the database stored in the dataset YAML; for # now we need to fetch the current ID. examples_id = (db.session.query(Database).filter_by( database_name="examples").one().id) dataset_info: Dict[str, Dict[str, Any]] = {} for file_name, config in configs.items(): if file_name.startswith("datasets/"): config["database_id"] = examples_id dataset = import_dataset(session, config, overwrite=overwrite) dataset_info[str(dataset.uuid)] = { "datasource_id": dataset.id, "datasource_type": "view" if dataset.is_sqllab_view else "table", "datasource_name": dataset.table_name, } # import charts chart_ids: Dict[str, int] = {} for file_name, config in configs.items(): if file_name.startswith("charts/"): # update datasource id, type, and name config.update(dataset_info[config["dataset_uuid"]]) chart = import_chart(session, config, overwrite=overwrite) chart_ids[str(chart.uuid)] = chart.id # store the existing relationship between dashboards and charts existing_relationships = session.execute( select( [dashboard_slices.c.dashboard_id, dashboard_slices.c.slice_id])).fetchall() # import dashboards dashboard_chart_ids: List[Tuple[int, int]] = [] for file_name, config in configs.items(): if file_name.startswith("dashboards/"): config = update_id_refs(config, chart_ids) dashboard = import_dashboard(session, config, overwrite=overwrite) for uuid in find_chart_uuids(config["position"]): chart_id = chart_ids[uuid] if (dashboard.id, chart_id) not in existing_relationships: dashboard_chart_ids.append((dashboard.id, chart_id)) # set ref in the dashboard_slices table values = [{ "dashboard_id": dashboard_id, "slice_id": chart_id } for (dashboard_id, chart_id) in dashboard_chart_ids] # pylint: disable=no-value-for-parameter (sqlalchemy/issues/4656) session.execute(dashboard_slices.insert(), values)
def _import( session: Session, configs: Dict[str, Any], overwrite: bool = False, force_data: bool = False, ) -> None: # import databases database_ids: Dict[str, int] = {} for file_name, config in configs.items(): if file_name.startswith("databases/"): database = import_database(session, config, overwrite=overwrite) database_ids[str(database.uuid)] = database.id # import datasets # If database_uuid is not in the list of UUIDs it means that the examples # database was created before its UUID was frozen, so it has a random UUID. # We need to determine its ID so we can point the dataset to it. examples_db = ( db.session.query(Database).filter_by(database_name="examples").first() ) dataset_info: Dict[str, Dict[str, Any]] = {} for file_name, config in configs.items(): if file_name.startswith("datasets/"): # find the ID of the corresponding database if config["database_uuid"] not in database_ids: if examples_db is None: raise Exception("Cannot find examples database") config["database_id"] = examples_db.id else: config["database_id"] = database_ids[config["database_uuid"]] dataset = import_dataset( session, config, overwrite=overwrite, force_data=force_data ) dataset_info[str(dataset.uuid)] = { "datasource_id": dataset.id, "datasource_type": "view" if dataset.is_sqllab_view else "table", "datasource_name": dataset.table_name, } # import charts chart_ids: Dict[str, int] = {} for file_name, config in configs.items(): if file_name.startswith("charts/"): # update datasource id, type, and name config.update(dataset_info[config["dataset_uuid"]]) chart = import_chart(session, config, overwrite=overwrite) chart_ids[str(chart.uuid)] = chart.id # store the existing relationship between dashboards and charts existing_relationships = session.execute( select([dashboard_slices.c.dashboard_id, dashboard_slices.c.slice_id]) ).fetchall() # import dashboards dashboard_chart_ids: List[Tuple[int, int]] = [] for file_name, config in configs.items(): if file_name.startswith("dashboards/"): config = update_id_refs(config, chart_ids, dataset_info) dashboard = import_dashboard(session, config, overwrite=overwrite) dashboard.published = True for uuid in find_chart_uuids(config["position"]): chart_id = chart_ids[uuid] if (dashboard.id, chart_id) not in existing_relationships: dashboard_chart_ids.append((dashboard.id, chart_id)) # set ref in the dashboard_slices table values = [ {"dashboard_id": dashboard_id, "slice_id": chart_id} for (dashboard_id, chart_id) in dashboard_chart_ids ] # pylint: disable=no-value-for-parameter # sqlalchemy/issues/4656 session.execute(dashboard_slices.insert(), values)
def _import( # pylint: disable=arguments-differ, too-many-locals, too-many-branches session: Session, configs: Dict[str, Any], overwrite: bool = False, force_data: bool = False, ) -> None: # import databases database_ids: Dict[str, int] = {} for file_name, config in configs.items(): if file_name.startswith("databases/"): database = import_database(session, config, overwrite=overwrite) database_ids[str(database.uuid)] = database.id # import datasets # If database_uuid is not in the list of UUIDs it means that the examples # database was created before its UUID was frozen, so it has a random UUID. # We need to determine its ID so we can point the dataset to it. examples_db = get_example_database() dataset_info: Dict[str, Dict[str, Any]] = {} for file_name, config in configs.items(): if file_name.startswith("datasets/"): # find the ID of the corresponding database if config["database_uuid"] not in database_ids: if examples_db is None: raise Exception("Cannot find examples database") config["database_id"] = examples_db.id else: config["database_id"] = database_ids[config["database_uuid"]] # set schema if config["schema"] is None: config["schema"] = get_example_default_schema() dataset = import_dataset( session, config, overwrite=overwrite, force_data=force_data ) try: dataset = import_dataset( session, config, overwrite=overwrite, force_data=force_data ) except MultipleResultsFound: # Multiple result can be found for datasets. There was a bug in # load-examples that resulted in datasets being loaded with a NULL # schema. Users could then add a new dataset with the same name in # the correct schema, resulting in duplicates, since the uniqueness # constraint was not enforced correctly in the application logic. # See https://github.com/apache/superset/issues/16051. continue dataset_info[str(dataset.uuid)] = { "datasource_id": dataset.id, "datasource_type": "table", "datasource_name": dataset.table_name, } # import charts chart_ids: Dict[str, int] = {} for file_name, config in configs.items(): if ( file_name.startswith("charts/") and config["dataset_uuid"] in dataset_info ): # update datasource id, type, and name config.update(dataset_info[config["dataset_uuid"]]) chart = import_chart(session, config, overwrite=overwrite) chart_ids[str(chart.uuid)] = chart.id # store the existing relationship between dashboards and charts existing_relationships = session.execute( select([dashboard_slices.c.dashboard_id, dashboard_slices.c.slice_id]) ).fetchall() # import dashboards dashboard_chart_ids: List[Tuple[int, int]] = [] for file_name, config in configs.items(): if file_name.startswith("dashboards/"): try: config = update_id_refs(config, chart_ids, dataset_info) except KeyError: continue dashboard = import_dashboard(session, config, overwrite=overwrite) dashboard.published = True for uuid in find_chart_uuids(config["position"]): chart_id = chart_ids[uuid] if (dashboard.id, chart_id) not in existing_relationships: dashboard_chart_ids.append((dashboard.id, chart_id)) # set ref in the dashboard_slices table values = [ {"dashboard_id": dashboard_id, "slice_id": chart_id} for (dashboard_id, chart_id) in dashboard_chart_ids ] session.execute(dashboard_slices.insert(), values)
def _import(session: Session, configs: Dict[str, Any], overwrite: bool = False) -> None: # discover charts associated with dashboards chart_uuids: Set[str] = set() for file_name, config in configs.items(): if file_name.startswith("dashboards/"): chart_uuids.update(find_chart_uuids(config["position"])) # discover datasets associated with charts dataset_uuids: Set[str] = set() for file_name, config in configs.items(): if file_name.startswith( "charts/") and config["uuid"] in chart_uuids: dataset_uuids.add(config["dataset_uuid"]) # discover databases associated with datasets database_uuids: Set[str] = set() for file_name, config in configs.items(): if file_name.startswith( "datasets/") and config["uuid"] in dataset_uuids: database_uuids.add(config["database_uuid"]) # import related databases database_ids: Dict[str, int] = {} for file_name, config in configs.items(): if file_name.startswith( "databases/") and config["uuid"] in database_uuids: database = import_database(session, config, overwrite=False) database_ids[str(database.uuid)] = database.id # import datasets with the correct parent ref dataset_info: Dict[str, Dict[str, Any]] = {} for file_name, config in configs.items(): if (file_name.startswith("datasets/") and config["database_uuid"] in database_ids): config["database_id"] = database_ids[config["database_uuid"]] dataset = import_dataset(session, config, overwrite=False) dataset_info[str(dataset.uuid)] = { "datasource_id": dataset.id, "datasource_type": "view" if dataset.is_sqllab_view else "table", "datasource_name": dataset.table_name, } # import charts with the correct parent ref chart_ids: Dict[str, int] = {} for file_name, config in configs.items(): if (file_name.startswith("charts/") and config["dataset_uuid"] in dataset_info): # update datasource id, type, and name config.update(dataset_info[config["dataset_uuid"]]) chart = import_chart(session, config, overwrite=False) chart_ids[str(chart.uuid)] = chart.id # store the existing relationship between dashboards and charts existing_relationships = session.execute( select( [dashboard_slices.c.dashboard_id, dashboard_slices.c.slice_id])).fetchall() # import dashboards dashboard_chart_ids: List[Tuple[int, int]] = [] for file_name, config in configs.items(): if file_name.startswith("dashboards/"): config = update_id_refs(config, chart_ids) dashboard = import_dashboard(session, config, overwrite=overwrite) for uuid in find_chart_uuids(config["position"]): if uuid not in chart_ids: break chart_id = chart_ids[uuid] if (dashboard.id, chart_id) not in existing_relationships: dashboard_chart_ids.append((dashboard.id, chart_id)) # set ref in the dashboard_slices table values = [{ "dashboard_id": dashboard_id, "slice_id": chart_id } for (dashboard_id, chart_id) in dashboard_chart_ids] # pylint: disable=no-value-for-parameter (sqlalchemy/issues/4656) session.execute(dashboard_slices.insert(), values)