def test_import_slices_for_non_existent_table(self): with self.assertRaises(AttributeError): Slice.import_obj( self.create_slice("Import Me 3", id=10004, table_name="non_existent"), None, )
def test_import_slices_override(self): slc = self.create_slice("Import Me New", id=10005) slc_1_id = Slice.import_obj(slc, None, import_time=1990) slc.slice_name = "Import Me New" imported_slc_1 = self.get_slice(slc_1_id) slc_2 = self.create_slice("Import Me New", id=10005) slc_2_id = Slice.import_obj(slc_2, imported_slc_1, import_time=1990) self.assertEqual(slc_1_id, slc_2_id) imported_slc_2 = self.get_slice(slc_2_id) self.assert_slice_equals(slc, imported_slc_2)
def test_import_2_slices_for_same_table(self): table_id = self.get_table_by_name("wb_health_population").id # table_id != 666, import func will have to find the table slc_1 = self.create_slice("Import Me 1", ds_id=666, id=10002) slc_id_1 = Slice.import_obj(slc_1, None) slc_2 = self.create_slice("Import Me 2", ds_id=666, id=10003) slc_id_2 = Slice.import_obj(slc_2, None) imported_slc_1 = self.get_slice(slc_id_1) imported_slc_2 = self.get_slice(slc_id_2) self.assertEqual(table_id, imported_slc_1.datasource_id) self.assert_slice_equals(slc_1, imported_slc_1) self.assertEqual(imported_slc_1.datasource.perm, imported_slc_1.perm) self.assertEqual(table_id, imported_slc_2.datasource_id) self.assert_slice_equals(slc_2, imported_slc_2) self.assertEqual(imported_slc_2.datasource.perm, imported_slc_2.perm)
def test_import_1_slice(self): expected_slice = self.create_slice("Import Me", id=10001) slc_id = Slice.import_obj(expected_slice, None, import_time=1989) slc = self.get_slice(slc_id) self.assertEqual(slc.datasource.perm, slc.perm) self.assert_slice_equals(expected_slice, slc) table_id = self.get_table_by_name("wb_health_population").id self.assertEqual(table_id, self.get_slice(slc_id).datasource_id)
def import_obj( # pylint: disable=too-many-locals,too-many-branches,too-many-statements cls, dashboard_to_import: "Dashboard", import_time: Optional[int] = None ) -> int: """Imports the dashboard from the object to the database. Once dashboard is imported, json_metadata field is extended and stores remote_id and import_time. It helps to decide if the dashboard has to be overridden or just copies over. Slices that belong to this dashboard will be wired to existing tables. This function can be used to import/export dashboards between multiple superset instances. Audit metadata isn't copied over. """ def alter_positions( dashboard: Dashboard, old_to_new_slc_id_dict: Dict[int, int] ) -> None: """ Updates slice_ids in the position json. Sample position_json data: { "DASHBOARD_VERSION_KEY": "v2", "DASHBOARD_ROOT_ID": { "type": "DASHBOARD_ROOT_TYPE", "id": "DASHBOARD_ROOT_ID", "children": ["DASHBOARD_GRID_ID"] }, "DASHBOARD_GRID_ID": { "type": "DASHBOARD_GRID_TYPE", "id": "DASHBOARD_GRID_ID", "children": ["DASHBOARD_CHART_TYPE-2"] }, "DASHBOARD_CHART_TYPE-2": { "type": "CHART", "id": "DASHBOARD_CHART_TYPE-2", "children": [], "meta": { "width": 4, "height": 50, "chartId": 118 } }, } """ position_data = json.loads(dashboard.position_json) position_json = position_data.values() for value in position_json: if ( isinstance(value, dict) and value.get("meta") and value.get("meta", {}).get("chartId") ): old_slice_id = value["meta"]["chartId"] if old_slice_id in old_to_new_slc_id_dict: value["meta"]["chartId"] = old_to_new_slc_id_dict[old_slice_id] dashboard.position_json = json.dumps(position_data) logger.info( "Started import of the dashboard: %s", dashboard_to_import.to_json() ) session = db.session logger.info("Dashboard has %d slices", len(dashboard_to_import.slices)) # copy slices object as Slice.import_slice will mutate the slice # and will remove the existing dashboard - slice association slices = copy(dashboard_to_import.slices) old_json_metadata = json.loads(dashboard_to_import.json_metadata or "{}") old_to_new_slc_id_dict: Dict[int, int] = {} new_timed_refresh_immune_slices = [] new_expanded_slices = {} new_filter_scopes = {} i_params_dict = dashboard_to_import.params_dict remote_id_slice_map = { slc.params_dict["remote_id"]: slc for slc in session.query(Slice).all() if "remote_id" in slc.params_dict } for slc in slices: logger.info( "Importing slice %s from the dashboard: %s", slc.to_json(), dashboard_to_import.dashboard_title, ) remote_slc = remote_id_slice_map.get(slc.id) new_slc_id = Slice.import_obj(slc, remote_slc, import_time=import_time) old_to_new_slc_id_dict[slc.id] = new_slc_id # update json metadata that deals with slice ids new_slc_id_str = "{}".format(new_slc_id) old_slc_id_str = "{}".format(slc.id) if ( "timed_refresh_immune_slices" in i_params_dict and old_slc_id_str in i_params_dict["timed_refresh_immune_slices"] ): new_timed_refresh_immune_slices.append(new_slc_id_str) if ( "expanded_slices" in i_params_dict and old_slc_id_str in i_params_dict["expanded_slices"] ): new_expanded_slices[new_slc_id_str] = i_params_dict["expanded_slices"][ old_slc_id_str ] # since PR #9109, filter_immune_slices and filter_immune_slice_fields # are converted to filter_scopes # but dashboard create from import may still have old dashboard filter metadata # here we convert them to new filter_scopes metadata first filter_scopes = {} if ( "filter_immune_slices" in i_params_dict or "filter_immune_slice_fields" in i_params_dict ): filter_scopes = convert_filter_scopes(old_json_metadata, slices) if "filter_scopes" in i_params_dict: filter_scopes = old_json_metadata.get("filter_scopes") # then replace old slice id to new slice id: if filter_scopes: new_filter_scopes = copy_filter_scopes( old_to_new_slc_id_dict=old_to_new_slc_id_dict, old_filter_scopes=filter_scopes, ) # override the dashboard existing_dashboard = None for dash in session.query(Dashboard).all(): if ( "remote_id" in dash.params_dict and dash.params_dict["remote_id"] == dashboard_to_import.id ): existing_dashboard = dash dashboard_to_import = dashboard_to_import.copy() dashboard_to_import.id = None dashboard_to_import.reset_ownership() # position_json can be empty for dashboards # with charts added from chart-edit page and without re-arranging if dashboard_to_import.position_json: alter_positions(dashboard_to_import, old_to_new_slc_id_dict) dashboard_to_import.alter_params(import_time=import_time) dashboard_to_import.remove_params(param_to_remove="filter_immune_slices") dashboard_to_import.remove_params(param_to_remove="filter_immune_slice_fields") if new_filter_scopes: dashboard_to_import.alter_params(filter_scopes=new_filter_scopes) if new_expanded_slices: dashboard_to_import.alter_params(expanded_slices=new_expanded_slices) if new_timed_refresh_immune_slices: dashboard_to_import.alter_params( timed_refresh_immune_slices=new_timed_refresh_immune_slices ) new_slices = ( session.query(Slice) .filter(Slice.id.in_(old_to_new_slc_id_dict.values())) .all() ) if existing_dashboard: existing_dashboard.override(dashboard_to_import) existing_dashboard.slices = new_slices session.flush() return existing_dashboard.id dashboard_to_import.slices = new_slices session.add(dashboard_to_import) session.flush() return dashboard_to_import.id # type: ignore