def import_obj( # pylint: disable=too-many-locals,too-many-branches,too-many-statements cls, dashboard_to_import: "Dashboard", import_time: Optional[int] = None ) -> int: """Imports the dashboard from the object to the database. Once dashboard is imported, json_metadata field is extended and stores remote_id and import_time. It helps to decide if the dashboard has to be overridden or just copies over. Slices that belong to this dashboard will be wired to existing tables. This function can be used to import/export dashboards between multiple superset instances. Audit metadata isn't copied over. """ def alter_positions( dashboard: Dashboard, old_to_new_slc_id_dict: Dict[int, int] ) -> None: """ Updates slice_ids in the position json. Sample position_json data: { "DASHBOARD_VERSION_KEY": "v2", "DASHBOARD_ROOT_ID": { "type": "DASHBOARD_ROOT_TYPE", "id": "DASHBOARD_ROOT_ID", "children": ["DASHBOARD_GRID_ID"] }, "DASHBOARD_GRID_ID": { "type": "DASHBOARD_GRID_TYPE", "id": "DASHBOARD_GRID_ID", "children": ["DASHBOARD_CHART_TYPE-2"] }, "DASHBOARD_CHART_TYPE-2": { "type": "CHART", "id": "DASHBOARD_CHART_TYPE-2", "children": [], "meta": { "width": 4, "height": 50, "chartId": 118 } }, } """ position_data = json.loads(dashboard.position_json) position_json = position_data.values() for value in position_json: if ( isinstance(value, dict) and value.get("meta") and value.get("meta", {}).get("chartId") ): old_slice_id = value["meta"]["chartId"] if old_slice_id in old_to_new_slc_id_dict: value["meta"]["chartId"] = old_to_new_slc_id_dict[old_slice_id] dashboard.position_json = json.dumps(position_data) logger.info( "Started import of the dashboard: %s", dashboard_to_import.to_json() ) session = db.session logger.info("Dashboard has %d slices", len(dashboard_to_import.slices)) # copy slices object as Slice.import_slice will mutate the slice # and will remove the existing dashboard - slice association slices = copy(dashboard_to_import.slices) old_json_metadata = json.loads(dashboard_to_import.json_metadata or "{}") old_to_new_slc_id_dict: Dict[int, int] = {} new_timed_refresh_immune_slices = [] new_expanded_slices = {} new_filter_scopes = {} i_params_dict = dashboard_to_import.params_dict remote_id_slice_map = { slc.params_dict["remote_id"]: slc for slc in session.query(Slice).all() if "remote_id" in slc.params_dict } for slc in slices: logger.info( "Importing slice %s from the dashboard: %s", slc.to_json(), dashboard_to_import.dashboard_title, ) remote_slc = remote_id_slice_map.get(slc.id) new_slc_id = Slice.import_obj(slc, remote_slc, import_time=import_time) old_to_new_slc_id_dict[slc.id] = new_slc_id # update json metadata that deals with slice ids new_slc_id_str = "{}".format(new_slc_id) old_slc_id_str = "{}".format(slc.id) if ( "timed_refresh_immune_slices" in i_params_dict and old_slc_id_str in i_params_dict["timed_refresh_immune_slices"] ): new_timed_refresh_immune_slices.append(new_slc_id_str) if ( "expanded_slices" in i_params_dict and old_slc_id_str in i_params_dict["expanded_slices"] ): new_expanded_slices[new_slc_id_str] = i_params_dict["expanded_slices"][ old_slc_id_str ] # since PR #9109, filter_immune_slices and filter_immune_slice_fields # are converted to filter_scopes # but dashboard create from import may still have old dashboard filter metadata # here we convert them to new filter_scopes metadata first filter_scopes = {} if ( "filter_immune_slices" in i_params_dict or "filter_immune_slice_fields" in i_params_dict ): filter_scopes = convert_filter_scopes(old_json_metadata, slices) if "filter_scopes" in i_params_dict: filter_scopes = old_json_metadata.get("filter_scopes") # then replace old slice id to new slice id: if filter_scopes: new_filter_scopes = copy_filter_scopes( old_to_new_slc_id_dict=old_to_new_slc_id_dict, old_filter_scopes=filter_scopes, ) # override the dashboard existing_dashboard = None for dash in session.query(Dashboard).all(): if ( "remote_id" in dash.params_dict and dash.params_dict["remote_id"] == dashboard_to_import.id ): existing_dashboard = dash dashboard_to_import = dashboard_to_import.copy() dashboard_to_import.id = None dashboard_to_import.reset_ownership() # position_json can be empty for dashboards # with charts added from chart-edit page and without re-arranging if dashboard_to_import.position_json: alter_positions(dashboard_to_import, old_to_new_slc_id_dict) dashboard_to_import.alter_params(import_time=import_time) dashboard_to_import.remove_params(param_to_remove="filter_immune_slices") dashboard_to_import.remove_params(param_to_remove="filter_immune_slice_fields") if new_filter_scopes: dashboard_to_import.alter_params(filter_scopes=new_filter_scopes) if new_expanded_slices: dashboard_to_import.alter_params(expanded_slices=new_expanded_slices) if new_timed_refresh_immune_slices: dashboard_to_import.alter_params( timed_refresh_immune_slices=new_timed_refresh_immune_slices ) new_slices = ( session.query(Slice) .filter(Slice.id.in_(old_to_new_slc_id_dict.values())) .all() ) if existing_dashboard: existing_dashboard.override(dashboard_to_import) existing_dashboard.slices = new_slices session.flush() return existing_dashboard.id dashboard_to_import.slices = new_slices session.add(dashboard_to_import) session.flush() return dashboard_to_import.id # type: ignore
def set_dash_metadata( # pylint: disable=too-many-locals dashboard: Dashboard, data: Dict[Any, Any], old_to_new_slice_ids: Optional[Dict[int, int]] = None, commit: bool = False, ) -> Dashboard: positions = data.get("positions") new_filter_scopes = {} md = dashboard.params_dict if positions is not None: # find slices in the position data slice_ids = [ value.get("meta", {}).get("chartId") for value in positions.values() if isinstance(value, dict) ] session = db.session() current_slices = session.query(Slice).filter( Slice.id.in_(slice_ids)).all() dashboard.slices = current_slices # add UUID to positions uuid_map = {slice.id: str(slice.uuid) for slice in current_slices} for obj in positions.values(): if (isinstance(obj, dict) and obj["type"] == "CHART" and obj["meta"]["chartId"]): chart_id = obj["meta"]["chartId"] obj["meta"]["uuid"] = uuid_map.get(chart_id) # remove leading and trailing white spaces in the dumped json dashboard.position_json = json.dumps(positions, indent=None, separators=(",", ":"), sort_keys=True) if "filter_scopes" in data: # replace filter_id and immune ids from old slice id to new slice id: # and remove slice ids that are not in dash anymore slc_id_dict: Dict[int, int] = {} if old_to_new_slice_ids: slc_id_dict = { old: new for old, new in old_to_new_slice_ids.items() if new in slice_ids } else: slc_id_dict = {sid: sid for sid in slice_ids} new_filter_scopes = copy_filter_scopes( old_to_new_slc_id_dict=slc_id_dict, old_filter_scopes=json.loads(data["filter_scopes"] or "{}") if isinstance(data["filter_scopes"], str) else data["filter_scopes"], ) default_filters_data = json.loads(data.get("default_filters", "{}")) applicable_filters = { key: v for key, v in default_filters_data.items() if int(key) in slice_ids } md["default_filters"] = json.dumps(applicable_filters) # positions have its own column, no need to store it in metadata md.pop("positions", None) # The css and dashboard_title properties are not part of the metadata # TODO (geido): remove by refactoring/deprecating save_dash endpoint if data.get("css") is not None: dashboard.css = data.get("css") if data.get("dashboard_title") is not None: dashboard.dashboard_title = data.get("dashboard_title") if new_filter_scopes: md["filter_scopes"] = new_filter_scopes else: md.pop("filter_scopes", None) md.setdefault("timed_refresh_immune_slices", []) if data.get("color_namespace") is None: md.pop("color_namespace", None) else: md["color_namespace"] = data.get("color_namespace") md["expanded_slices"] = data.get("expanded_slices", {}) md["refresh_frequency"] = data.get("refresh_frequency", 0) md["color_scheme"] = data.get("color_scheme", "") md["label_colors"] = data.get("label_colors", {}) md["shared_label_colors"] = data.get("shared_label_colors", {}) dashboard.json_metadata = json.dumps(md) if commit: db.session.commit() return dashboard
def set_dash_metadata( dashboard: Dashboard, data: Dict[Any, Any], old_to_new_slice_ids: Optional[Dict[int, int]] = None, ) -> None: positions = data["positions"] # find slices in the position data slice_ids = [ value.get("meta", {}).get("chartId") for value in positions.values() if isinstance(value, dict) ] session = db.session() current_slices = session.query(Slice).filter( Slice.id.in_(slice_ids)).all() dashboard.slices = current_slices # add UUID to positions uuid_map = {slice.id: str(slice.uuid) for slice in current_slices} for obj in positions.values(): if (isinstance(obj, dict) and obj["type"] == "CHART" and obj["meta"]["chartId"]): chart_id = obj["meta"]["chartId"] obj["meta"]["uuid"] = uuid_map.get(chart_id) # remove leading and trailing white spaces in the dumped json dashboard.position_json = json.dumps(positions, indent=None, separators=(",", ":"), sort_keys=True) md = dashboard.params_dict dashboard.css = data.get("css") dashboard.dashboard_title = data["dashboard_title"] if "timed_refresh_immune_slices" not in md: md["timed_refresh_immune_slices"] = [] new_filter_scopes = {} if "filter_scopes" in data: # replace filter_id and immune ids from old slice id to new slice id: # and remove slice ids that are not in dash anymore slc_id_dict: Dict[int, int] = {} if old_to_new_slice_ids: slc_id_dict = { old: new for old, new in old_to_new_slice_ids.items() if new in slice_ids } else: slc_id_dict = {sid: sid for sid in slice_ids} new_filter_scopes = copy_filter_scopes( old_to_new_slc_id_dict=slc_id_dict, old_filter_scopes=json.loads(data["filter_scopes"] or "{}"), ) if new_filter_scopes: md["filter_scopes"] = new_filter_scopes else: md.pop("filter_scopes", None) md["expanded_slices"] = data.get("expanded_slices", {}) md["refresh_frequency"] = data.get("refresh_frequency", 0) default_filters_data = json.loads(data.get("default_filters", "{}")) applicable_filters = { key: v for key, v in default_filters_data.items() if int(key) in slice_ids } md["default_filters"] = json.dumps(applicable_filters) md["color_scheme"] = data.get("color_scheme") if data.get("color_namespace"): md["color_namespace"] = data.get("color_namespace") if data.get("label_colors"): md["label_colors"] = data.get("label_colors") dashboard.json_metadata = json.dumps(md)
def set_dash_metadata( # pylint: disable=too-many-locals,too-many-branches,too-many-statements dashboard: Dashboard, data: Dict[Any, Any], old_to_new_slice_ids: Optional[Dict[int, int]] = None, ) -> None: positions = data["positions"] # find slices in the position data slice_ids = [] slice_id_to_name = {} for value in positions.values(): if isinstance(value, dict): try: slice_id = value["meta"]["chartId"] slice_ids.append(slice_id) slice_id_to_name[slice_id] = value["meta"]["sliceName"] except KeyError: pass session = db.session() current_slices = session.query(Slice).filter( Slice.id.in_(slice_ids)).all() dashboard.slices = current_slices # update slice names. this assumes user has permissions to update the slice # we allow user set slice name be empty string for slc in dashboard.slices: try: new_name = slice_id_to_name[slc.id] if slc.slice_name != new_name: slc.slice_name = new_name session.merge(slc) session.flush() except KeyError: pass # remove leading and trailing white spaces in the dumped json dashboard.position_json = json.dumps(positions, indent=None, separators=(",", ":"), sort_keys=True) md = dashboard.params_dict dashboard.css = data.get("css") dashboard.dashboard_title = data["dashboard_title"] if "timed_refresh_immune_slices" not in md: md["timed_refresh_immune_slices"] = [] new_filter_scopes = {} if "filter_scopes" in data: # replace filter_id and immune ids from old slice id to new slice id: # and remove slice ids that are not in dash anymore slc_id_dict: Dict[int, int] = {} if old_to_new_slice_ids: slc_id_dict = { old: new for old, new in old_to_new_slice_ids.items() if new in slice_ids } else: slc_id_dict = {sid: sid for sid in slice_ids} new_filter_scopes = copy_filter_scopes( old_to_new_slc_id_dict=slc_id_dict, old_filter_scopes=json.loads(data["filter_scopes"] or "{}"), ) if new_filter_scopes: md["filter_scopes"] = new_filter_scopes else: md.pop("filter_scopes", None) md["expanded_slices"] = data.get("expanded_slices", {}) md["refresh_frequency"] = data.get("refresh_frequency", 0) default_filters_data = json.loads(data.get("default_filters", "{}")) applicable_filters = { key: v for key, v in default_filters_data.items() if int(key) in slice_ids } md["default_filters"] = json.dumps(applicable_filters) md["color_scheme"] = data.get("color_scheme") if data.get("color_namespace"): md["color_namespace"] = data.get("color_namespace") if data.get("label_colors"): md["label_colors"] = data.get("label_colors") dashboard.json_metadata = json.dumps(md)