def panda_process(indf): """ Build a data set from a table using pandas. This attempts to respect categorical data input by letting pandas.read_csv infer the type """ result = Data() for name, column in indf.iteritems(): if (column.dtype == np.object) | (column.dtype == np.bool): # try to salvage numerical data coerced = column.convert_objects(convert_numeric=True) if (coerced.dtype != column.dtype) and coerced.isnull().mean() < 0.4: c = Component(coerced.values) else: # pandas has a 'special' nan implementation and this doesn't # play well with np.unique c = CategoricalComponent(column.fillna('')) else: c = Component(column.values) # convert header to string - in some cases if the first row contains # numbers, these are cast to numerical types, so we want to change that # here. if not isinstance(name, six.string_types): name = str(name) # strip off leading # name = name.strip() if name.startswith('#'): name = name[1:].strip() result.add_component(c, name) return result
def test_histogram_data(): data = Data(label="Test Data") comp_a = Component(np.random.uniform(size=500)) comp_b = Component(np.random.normal(size=500)) data.add_component(comp_a, 'uniform') data.add_component(comp_b, 'normal') return data
def test_to_ccddata_invalid(): data = Data(label='not-an-image') data.add_component(Component(np.array([3.4, 2.3, -1.1, 0.3]), units='Jy'), 'x') with pytest.raises(ValueError) as exc: data.get_object(CCDData, attribute=data.id['x']) assert exc.value.args[ 0] == 'Only 2-dimensional datasets can be converted to CCDData' class FakeCoordinates(Coordinates): def pixel_to_world_values(self, *pixel): raise NotImplementedError() def world_to_pixel_values(self, *pixel): raise NotImplementedError() coords = FakeCoordinates(n_dim=2) coords.low_level_wcs = coords data = Data(label='image-with-custom-coords', coords=coords) data.add_component(Component(np.array([[3, 4], [4, 5]]), units='Jy'), 'x') with pytest.raises(TypeError) as exc: data.get_object(CCDData, attribute=data.id['x']) assert exc.value.args[ 0] == 'data.coords should be an instance of Coordinates or WCS'
def test_image(): data = Data(label="Test Image") comp_a = Component(np.ones((25, 25))) data.add_component(comp_a, 'test_1') comp_b = Component(np.zeros((25, 25))) data.add_component(comp_b, 'test_2') return data
def setup_method(self, method): # Set up simple spectral WCS wcs_1d = WCS(naxis=1) wcs_1d.wcs.ctype = ['VELO-LSR'] wcs_1d.wcs.set() # Set up a spectral cube WCS wcs_3d = WCS(naxis=3) wcs_3d.wcs.ctype = ['RA---TAN', 'DEC--TAN', 'VELO-LSR'] wcs_3d.wcs.set() # Set up glue Coordinates object coords_1d = WCSCoordinates(wcs=wcs_1d) coords_3d = WCSCoordinates(wcs=wcs_3d) self.data_1d = Data(label='spectrum', coords=coords_1d) self.data_3d = Data(label='spectrum', coords=coords_3d) # FIXME: there should be an easier way to do this in glue x = np.array([3.4, 2.3, -1.1, 0.3]) y = np.array([3.2, 3.3, 3.4, 3.5]) self.data_1d.add_component(Component(x, units='Jy'), 'x') self.data_1d.add_component(Component(y, units='Jy'), 'y') self.data_3d.add_component(Component(np.broadcast_to(x, (6, 5, 4)), units='Jy'), 'x') self.data_3d.add_component(Component(np.broadcast_to(x, (6, 5, 4))), 'y') self.app = GlueApplication() self.session = self.app.session self.hub = self.session.hub self.data_collection = self.session.data_collection self.data_collection.append(self.data_1d) self.data_collection.append(self.data_3d)
def test_data(): data = Data(label="Test Data 1") data2 = Data(label="Teset Data 2") comp_a = Component(np.array([1, 2, 3])) comp_b = Component(np.array([1, 2, 3])) comp_c = Component(np.array([2, 4, 6])) comp_d = Component(np.array([1, 3, 5])) data.add_component(comp_a, 'a') data.add_component(comp_b, 'b') data2.add_component(comp_c, 'c') data2.add_component(comp_d, 'd') return data, data2
def test_limits_inf(self): d = Data() x = Component(np.array([[1, 2], [np.infty, 4]])) y = Component(np.array([[2, 4], [-np.infty, 8]])) xid = d.add_component(x, 'x') yid = d.add_component(y, 'y') self.collect.append(d) self.client.add_layer(d) self.client.xatt = xid self.client.yatt = yid assert self.client._visible_limits(0) == (1, 4) assert self.client._visible_limits(1) == (2, 8)
def make_test_data(): data = Data(label="Test Cat Data 1") comp_x1 = Component(np.array([4, 5, 6, 3])) comp_y1 = Component(np.array([1, 2, 3, 2])) comp_z1 = Component(np.array([2, 3, 4, 1])) data.add_component(comp_x1, 'x_gal') data.add_component(comp_y1, 'y_gal') data.add_component(comp_z1, 'z_gal') return data
def make_test_data(): data = Data(label="Test Cube Data") np.random.seed(12345) for letter in 'abc': comp = Component(np.random.random((10, 10, 10))) data.add_component(comp, letter) # make sure one component key is primary data.add_component(Component(np.random.random((10, 10, 10))), 'PRIMARY') return data
def test_categorical_data(): data = Data(label="Test Cat Data 1") data2 = Data(label="Teset Cat Data 2") comp_x1 = CategoricalComponent(np.array(['a', 'a', 'b'])) comp_y1 = Component(np.array([1, 2, 3])) comp_x2 = CategoricalComponent(np.array(['c', 'a', 'b'])) comp_y2 = Component(np.array([1, 3, 5])) data.add_component(comp_x1, 'x1') data.add_component(comp_y1, 'y1') data2.add_component(comp_x2, 'x2') data2.add_component(comp_y2, 'y2') return data, data2
def test_to_spectrum1d(): # Set up simple spectral WCS wcs = WCS(naxis=1) wcs.wcs.ctype = ['VELO-LSR'] wcs.wcs.set() coords = WCSCoordinates(wcs=wcs) data = Data(label='spectrum', coords=coords) data.add_component(Component(np.array([3.4, 2.3, -1.1, 0.3]), units='Jy'), 'x') spec = data.get_object(Spectrum1D, attribute=data.id['x']) assert_quantity_allclose(spec.spectral_axis, [1, 2, 3, 4] * u.m / u.s) assert_quantity_allclose(spec.flux, [3.4, 2.3, -1.1, 0.3] * u.Jy) data.add_subset(data.id['x'] > 1, label='bright') spec_subset = data.get_subset_object(cls=Spectrum1D, subset_id=0, attribute=data.id['x']) assert_quantity_allclose(spec_subset.spectral_axis, [1, 2, 3, 4] * u.m / u.s) assert_quantity_allclose(spec_subset.flux, [3.4, 2.3, np.nan, np.nan] * u.Jy) assert_equal(spec_subset.mask, [1, 1, 0, 0])
def test_to_ccddata(with_wcs): if with_wcs: coords = WCS_CELESTIAL else: coords = None data = Data(label='image', coords=coords) data.add_component( Component(np.array([[3.4, 2.3], [-1.1, 0.3]]), units='Jy'), 'x') image = data.get_object(CCDData, attribute=data.id['x']) assert image.wcs is (WCS_CELESTIAL if with_wcs else None) assert_allclose(image.data, [[3.4, 2.3], [-1.1, 0.3]]) assert image.unit is u.Jy data.add_subset(data.id['x'] > 1, label='bright') image_subset = data.get_subset_object(cls=CCDData, subset_id=0, attribute=data.id['x']) assert image_subset.wcs is (WCS_CELESTIAL if with_wcs else None) assert_allclose(image_subset.data, [[3.4, 2.3], [-1.1, 0.3]]) assert image_subset.unit is u.Jy assert_equal(image_subset.mask, [[0, 0], [1, 1]])
def ds9_region(filename): reg = regions.read_ds9(filename) comp = Component(np.ones(len(reg), dtype='bool')) data = RegionData(label='Regions: {0}'.format(os.path.split(filename)[-1]), regions=reg) return data
def load_stacked_sequence(self, raster_data): for window, window_data in raster_data.items(): w_data = Data(label=f"{window.replace(' ', '_')}") w_data.coords = window_data.wcs w_data.add_component(Component(window_data.data), f"{window.replace(' ', '_')}") w_data.style = VisualAttributes(color='#7A617C') self.datasets.append(w_data)
def test_to_spectral_cube_invalid_ndim(): data = Data(label='not-a-spectral-cube') data.add_component(Component(np.array([3.4, 2.3, -1.1, 0.3]), units='Jy'), 'x') with pytest.raises(ValueError) as exc: data.get_object(SpectralCube, attribute=data.id['x']) assert exc.value.args[0] == ('Data object should have 3 or 4 dimensions in order ' 'to be converted to a SpectralCube object.')
def test_to_spectral_cube_missing_wcs(): data = Data(label='not-a-spectral-cube') values = np.random.random((4, 5, 3)) data.add_component(Component(values, units='Jy'), 'x') with pytest.raises(TypeError) as exc: data.get_object(SpectralCube, attribute=data.id['x']) assert exc.value.args[0] == ('data.coords should be an instance of BaseLowLevelWCS.')
def test_to_ccddata_unitless(): data = Data(label='image', coords=WCS_CELESTIAL) data.add_component(Component(np.array([[3.4, 2.3], [-1.1, 0.3]])), 'x') image = data.get_object(CCDData, attribute=data.id['x']) assert_allclose(image.data, [[3.4, 2.3], [-1.1, 0.3]]) assert image.unit is u.one
def test_to_spectrum1d_invalid(): data = Data(label='not-a-spectrum') data.add_component(Component(np.array([3.4, 2.3, -1.1, 0.3]), units='Jy'), 'x') with pytest.raises(TypeError) as exc: data.get_object(Spectrum1D, attribute=data.id['x']) assert exc.value.args[0] == ('data.coords should be an instance of WCS ' 'or SpectralCoordinates')
def test_ticks_go_back_after_changing(self): """ If you change to a categorical axis and then change back to a numeric, the axis ticks should fix themselves properly. """ data = Data() data.add_component(Component(np.arange(100)), 'y') data.add_component(CategoricalComponent(['a'] * 50 + ['b'] * 50), 'xcat') data.add_component(Component(2 * np.arange(100)), 'xcont') self.add_data(data=data) self.client.yatt = data.find_component_id('y') self.client.xatt = data.find_component_id('xcat') self.check_ticks(self.client.axes.xaxis, False, True) self.check_ticks(self.client.axes.yaxis, False, False) self.client.xatt = data.find_component_id('xcont') self.check_ticks(self.client.axes.yaxis, False, False) self.check_ticks(self.client.axes.xaxis, False, False)
def test_to_spectral_cube_unitless(spectral_cube_wcs): data = Data(label='spectral_cube', coords=spectral_cube_wcs) values = np.random.random((4, 5, 3)) data.add_component(Component(values), 'x') spec = data.get_object(SpectralCube, attribute=data.id['x']) assert_quantity_allclose(spec.spectral_axis, [1, 2, 3, 4] * u.m / u.s) assert_quantity_allclose(spec.filled_data[...], values * u.one)
def make_test_data(): data = Data(label="Test Cat Data 1") np.random.seed(12345) for letter in 'abcdefxyz': comp = Component(np.random.random(100)) data.add_component(comp, letter) return data
def make_test_data(): data = Data(label="Test Cube Data") np.random.seed(12345) for letter in 'abc': comp = Component(np.random.random((10, 10, 10))) data.add_component(comp, letter) return data
def test_to_spectral_cube_invalid_wcs(): wcs = WCS(naxis=3) data = Data(label='not-a-spectral-cube', coords=wcs) values = np.random.random((4, 5, 3)) data.add_component(Component(values, units='Jy'), 'x') with pytest.raises(ValueError) as exc: data.get_object(SpectralCube, attribute=data.id['x']) assert exc.value.args[0] == ('No celestial axes found in WCS') wcs.wcs.ctype = ['RA---TAN', 'DEC--TAN', ''] data = Data(label='not-a-spectral-cube', coords=wcs) values = np.random.random((4, 5, 3)) data.add_component(Component(values, units='Jy'), 'x') with pytest.raises(ValueError) as exc: data.get_object(SpectralCube, attribute=data.id['x']) assert exc.value.args[0] == ('No spectral axes found in WCS')
def load_sunpy_map(self, sunpy_map): sunpy_map_loaded = sunpy.map.Map(sunpy_map) label = 'sunpy-map-' + sunpy_map_loaded.name data = Data(label=label) data.coords = sunpy_map_loaded.wcs # preferred way, preserves more info in some cases data.meta = sunpy_map_loaded.meta data.add_component(Component(sunpy_map_loaded.data), sunpy_map_loaded.name) data.style = VisualAttributes(color='#FDB813', preferred_cmap=sunpy_map.cmap) self.datasets.append(data)
def test_to_spectrum1d_with_spectral_coordinates(): coords = SpectralCoordinates([1, 4, 10] * u.micron) data = Data(label='spectrum1d', coords=coords) data.add_component(Component(np.array([3, 4, 5]), units='Jy'), 'x') assert_allclose(data.coords.pixel2world([0, 0.5, 1, 1.5, 2]), [[1, 2.5, 4, 7, 10]]) spec = data.get_object(Spectrum1D, attribute=data.id['x']) assert_quantity_allclose(spec.spectral_axis, [1, 4, 10] * u.micron) assert_quantity_allclose(spec.flux, [3, 4, 5] * u.Jy)
def test_to_spectral_cube_default_attribute(spectral_cube_wcs): data = Data(label='spectral_cube', coords=spectral_cube_wcs) values = np.random.random((4, 5, 3)) with pytest.raises(ValueError) as exc: data.get_object(SpectralCube) assert exc.value.args[0] == 'Data object has no attributes.' data.add_component(Component(values, units='Jy'), 'x') spec = data.get_object(SpectralCube) assert_quantity_allclose(spec.filled_data[...], values * u.Jy) data.add_component(Component(values, units='Jy'), 'y') with pytest.raises(ValueError) as exc: data.get_object(SpectralCube) assert exc.value.args[0] == ('Data object has more than one attribute, so ' 'you will need to specify which one to use as ' 'the flux for the spectral cube using the attribute= ' 'keyword argument.')
def test_to_ccddata_default_attribute(): data = Data(label='image', coords=WCS_CELESTIAL) with pytest.raises(ValueError) as exc: data.get_object(CCDData) assert exc.value.args[0] == 'Data object has no attributes.' data.add_component(Component(np.array([[3, 4], [5, 6]]), units='Jy'), 'x') image = data.get_object(CCDData) assert_allclose(image.data, [[3, 4], [5, 6]]) assert image.unit is u.Jy data.add_component(Component(np.array([[3, 4], [5, 6]]), units='Jy'), 'y') with pytest.raises(ValueError) as exc: data.get_object(CCDData) assert exc.value.args[0] == ('Data object has more than one attribute, so ' 'you will need to specify which one to use as ' 'the flux for the spectrum using the attribute= ' 'keyword argument.')
def load_sji(self, sji): with fits.open(sji) as hdul: hdul.verify("fix") label = hdul[0].header['TDESC1'] + hdul[0].header['OBSID'] data = Data(label=label) data.coords = WCSCoordinates(hdul[0].header) data.meta = hdul[0].header preferred_cmap_name = 'IRIS ' + hdul[0].header['TDESC1'].replace( '_', ' ') data.style = VisualAttributes(preferred_cmap=preferred_cmap_name) data.add_component(Component(hdul[0].data), label) self.datasets.append(data)
def load_sequence(self, raster_data): for window, window_data in raster_data.items(): for i, scan_data in enumerate(window_data): w_data = Data( label= f"{window.replace(' ', '_')}-{scan_data.meta['OBSID']}-scan-{i}" ) w_data.coords = scan_data.wcs w_data.add_component(Component(scan_data.data), f"{window.replace(' ', '_')}-scan-{i}") w_data.meta = scan_data.meta w_data.style = VisualAttributes(color='#5A4FCF') self.datasets.append(w_data)
def test_high_cardinatility_timing(self): card = 50000 data = Data() card_data = [str(num) for num in range(card)] data.add_component(Component(np.arange(card * 5)), 'y') data.add_component(CategoricalComponent(np.repeat([card_data], 5)), 'xcat') self.add_data(data) comp = data.find_component_id('xcat') timer_func = partial(self.client._set_xydata, 'x', comp) timer = timeit(timer_func, number=1) assert timer < 3 # this is set for Travis speed
def panda_process(indf): """ Build a data set from a table using pandas. This attempts to respect categorical data input by letting pandas.read_csv infer the type """ result = Data() for name, column in indf.iteritems(): if (column.dtype == np.object) | (column.dtype == np.bool): # try to salvage numerical data try: coerced = pd.to_numeric(column, errors='coerce') except AttributeError: # pandas < 0.19 coerced = column.convert_objects(convert_numeric=True) if (coerced.dtype != column.dtype) and coerced.isnull().mean() < 0.4: c = Component(coerced.values) else: # pandas has a 'special' nan implementation and this doesn't # play well with np.unique c = CategoricalComponent(column.fillna('')) else: c = Component.autotyped(column.values) # convert header to string - in some cases if the first row contains # numbers, these are cast to numerical types, so we want to change that # here. if not isinstance(name, six.string_types): name = str(name) # strip off leading # name = name.strip() if name.startswith('#'): name = name[1:].strip() result.add_component(c, name) return result
def add_component(self, component, label, hidden=False): """ Add a new component to this data set. :param component: object to add. Can be a Component, array-like object, or ComponentLink :param label: The label. If this is a string, a new :class:`glue.core.component_id.ComponentID` with this label will be created and associated with the Component :type component: :class:`~glue.core.component.Component` or array-like :type label: :class:`str` or :class:`~glue.core.component_id.ComponentID` :raises: TypeError, if label is invalid ValueError if the component has an incompatible shape :returns: The ComponentID associated with the newly-added component """ if isinstance(component, ComponentLink): return self.add_component_link(component, label=label, hidden=hidden) if not isinstance(component, Component): component = Component.autotyped(component) if isinstance(component, DerivedComponent): if len(self._components) == 0: raise TypeError("Cannot add a derived component as a first component") component.set_parent(self) if not(self._check_can_add(component)): raise ValueError("The dimensions of component %s are " "incompatible with the dimensions of this data: " "%r vs %r" % (label, component.shape, self.shape)) if isinstance(label, ComponentID): component_id = label if component_id.parent is None: component_id.parent = self else: component_id = ComponentID(label, hidden=hidden, parent=self) if len(self._components) == 0: self._create_pixel_and_world_components(ndim=component.ndim) # In some cases, such as when loading a session, we actually disable the # auto-creation of pixel and world coordinates, so the first component # may be a coordinate component with no shape. Therefore we only set the # shape once a component has a valid shape rather than strictly on the # first component. if self._shape == () and component.shape != (): self._shape = component.shape is_present = component_id in self._components self._components[component_id] = component if self.hub and not is_present: msg = DataAddComponentMessage(self, component_id) self.hub.broadcast(msg) msg = ComponentsChangedMessage(self) self.hub.broadcast(msg) return component_id
def add_component(self, component, label, hidden=False): """ Add a new component to this data set. :param component: object to add. Can be a Component, array-like object, or ComponentLink :param label: The label. If this is a string, a new :class:`glue.core.component_id.ComponentID` with this label will be created and associated with the Component :type component: :class:`~glue.core.component.Component` or array-like :type label: :class:`str` or :class:`~glue.core.component_id.ComponentID` :raises: TypeError, if label is invalid ValueError if the component has an incompatible shape :returns: The ComponentID associated with the newly-added component """ if isinstance(component, ComponentLink): component = DerivedComponent(self, component) if not isinstance(component, Component): component = Component.autotyped(component) if isinstance(component, DerivedComponent): component.set_parent(self) if not(self._check_can_add(component)): raise ValueError("The dimensions of component %s are " "incompatible with the dimensions of this data: " "%r vs %r" % (label, component.shape, self.shape)) if isinstance(label, ComponentID): component_id = label else: component_id = ComponentID(label, hidden=hidden) is_present = component_id in self._components self._components[component_id] = component first_component = len(self._components) == 1 if first_component: if isinstance(component, DerivedComponent): raise TypeError("Cannot add a derived component as " "first component") self._shape = component.shape self._create_pixel_and_world_components() if self.hub and (not is_present): msg = DataAddComponentMessage(self, component_id) self.hub.broadcast(msg) msg = ComponentsChangedMessage(self) self.hub.broadcast(msg) return component_id