def upgrade(): bind = op.get_bind() session = db.Session(bind=bind) dashboards = session.query(Dashboard).all() for i, dashboard in enumerate(dashboards): print(f"scanning dashboard ({i + 1}/{len(dashboards)}) >>>>") try: json_metadata = json.loads(dashboard.json_metadata or "{}") if "filter_scopes" in json_metadata: continue filters = [ slice for slice in dashboard.slices if slice.viz_type == "filter_box" ] # if dashboard has filter_box if filters: filter_scopes = convert_filter_scopes(json_metadata, filters) json_metadata["filter_scopes"] = filter_scopes logging.info( f"Adding filter_scopes for dashboard {dashboard.id}: {json.dumps(filter_scopes)}" ) json_metadata.pop("filter_immune_slices", None) json_metadata.pop("filter_immune_slice_fields", None) if json_metadata: dashboard.json_metadata = json.dumps(json_metadata, indent=None, separators=(",", ":"), sort_keys=True) else: dashboard.json_metadata = None session.merge(dashboard) except Exception as e: logging.exception(f"dashboard {dashboard.id} has error: {e}") session.commit() session.close()
def import_obj( # pylint: disable=too-many-locals,too-many-branches,too-many-statements cls, dashboard_to_import: "Dashboard", import_time: Optional[int] = None ) -> int: """Imports the dashboard from the object to the database. Once dashboard is imported, json_metadata field is extended and stores remote_id and import_time. It helps to decide if the dashboard has to be overridden or just copies over. Slices that belong to this dashboard will be wired to existing tables. This function can be used to import/export dashboards between multiple superset instances. Audit metadata isn't copied over. """ def alter_positions( dashboard: Dashboard, old_to_new_slc_id_dict: Dict[int, int] ) -> None: """ Updates slice_ids in the position json. Sample position_json data: { "DASHBOARD_VERSION_KEY": "v2", "DASHBOARD_ROOT_ID": { "type": "DASHBOARD_ROOT_TYPE", "id": "DASHBOARD_ROOT_ID", "children": ["DASHBOARD_GRID_ID"] }, "DASHBOARD_GRID_ID": { "type": "DASHBOARD_GRID_TYPE", "id": "DASHBOARD_GRID_ID", "children": ["DASHBOARD_CHART_TYPE-2"] }, "DASHBOARD_CHART_TYPE-2": { "type": "CHART", "id": "DASHBOARD_CHART_TYPE-2", "children": [], "meta": { "width": 4, "height": 50, "chartId": 118 } }, } """ position_data = json.loads(dashboard.position_json) position_json = position_data.values() for value in position_json: if ( isinstance(value, dict) and value.get("meta") and value.get("meta", {}).get("chartId") ): old_slice_id = value["meta"]["chartId"] if old_slice_id in old_to_new_slc_id_dict: value["meta"]["chartId"] = old_to_new_slc_id_dict[old_slice_id] dashboard.position_json = json.dumps(position_data) logger.info( "Started import of the dashboard: %s", dashboard_to_import.to_json() ) session = db.session logger.info("Dashboard has %d slices", len(dashboard_to_import.slices)) # copy slices object as Slice.import_slice will mutate the slice # and will remove the existing dashboard - slice association slices = copy(dashboard_to_import.slices) old_json_metadata = json.loads(dashboard_to_import.json_metadata or "{}") old_to_new_slc_id_dict: Dict[int, int] = {} new_timed_refresh_immune_slices = [] new_expanded_slices = {} new_filter_scopes = {} i_params_dict = dashboard_to_import.params_dict remote_id_slice_map = { slc.params_dict["remote_id"]: slc for slc in session.query(Slice).all() if "remote_id" in slc.params_dict } for slc in slices: logger.info( "Importing slice %s from the dashboard: %s", slc.to_json(), dashboard_to_import.dashboard_title, ) remote_slc = remote_id_slice_map.get(slc.id) new_slc_id = Slice.import_obj(slc, remote_slc, import_time=import_time) old_to_new_slc_id_dict[slc.id] = new_slc_id # update json metadata that deals with slice ids new_slc_id_str = "{}".format(new_slc_id) old_slc_id_str = "{}".format(slc.id) if ( "timed_refresh_immune_slices" in i_params_dict and old_slc_id_str in i_params_dict["timed_refresh_immune_slices"] ): new_timed_refresh_immune_slices.append(new_slc_id_str) if ( "expanded_slices" in i_params_dict and old_slc_id_str in i_params_dict["expanded_slices"] ): new_expanded_slices[new_slc_id_str] = i_params_dict["expanded_slices"][ old_slc_id_str ] # since PR #9109, filter_immune_slices and filter_immune_slice_fields # are converted to filter_scopes # but dashboard create from import may still have old dashboard filter metadata # here we convert them to new filter_scopes metadata first filter_scopes = {} if ( "filter_immune_slices" in i_params_dict or "filter_immune_slice_fields" in i_params_dict ): filter_scopes = convert_filter_scopes(old_json_metadata, slices) if "filter_scopes" in i_params_dict: filter_scopes = old_json_metadata.get("filter_scopes") # then replace old slice id to new slice id: if filter_scopes: new_filter_scopes = copy_filter_scopes( old_to_new_slc_id_dict=old_to_new_slc_id_dict, old_filter_scopes=filter_scopes, ) # override the dashboard existing_dashboard = None for dash in session.query(Dashboard).all(): if ( "remote_id" in dash.params_dict and dash.params_dict["remote_id"] == dashboard_to_import.id ): existing_dashboard = dash dashboard_to_import = dashboard_to_import.copy() dashboard_to_import.id = None dashboard_to_import.reset_ownership() # position_json can be empty for dashboards # with charts added from chart-edit page and without re-arranging if dashboard_to_import.position_json: alter_positions(dashboard_to_import, old_to_new_slc_id_dict) dashboard_to_import.alter_params(import_time=import_time) dashboard_to_import.remove_params(param_to_remove="filter_immune_slices") dashboard_to_import.remove_params(param_to_remove="filter_immune_slice_fields") if new_filter_scopes: dashboard_to_import.alter_params(filter_scopes=new_filter_scopes) if new_expanded_slices: dashboard_to_import.alter_params(expanded_slices=new_expanded_slices) if new_timed_refresh_immune_slices: dashboard_to_import.alter_params( timed_refresh_immune_slices=new_timed_refresh_immune_slices ) new_slices = ( session.query(Slice) .filter(Slice.id.in_(old_to_new_slc_id_dict.values())) .all() ) if existing_dashboard: existing_dashboard.override(dashboard_to_import) existing_dashboard.slices = new_slices session.flush() return existing_dashboard.id dashboard_to_import.slices = new_slices session.add(dashboard_to_import) session.flush() return dashboard_to_import.id # type: ignore