def panda_process(indf):
"""
Build a data set from a table using pandas. This attempts to respect
categorical data input by letting pandas.read_csv infer the type
"""
result = Data()
for name, column in indf.iteritems():
if (column.dtype == np.object) | (column.dtype == np.bool):
# try to salvage numerical data
coerced = column.convert_objects(convert_numeric=True)
if (coerced.dtype !=
column.dtype) and coerced.isnull().mean() < 0.4:
c = Component(coerced.values)
else:
# pandas has a 'special' nan implementation and this doesn't
# play well with np.unique
c = CategoricalComponent(column.fillna(''))
else:
c = Component(column.values)
# convert header to string - in some cases if the first row contains
# numbers, these are cast to numerical types, so we want to change that
# here.
if not isinstance(name, six.string_types):
name = str(name)
# strip off leading #
name = name.strip()
if name.startswith('#'):
name = name[1:].strip()
result.add_component(c, name)
return result
def test_histogram_data():
data = Data(label="Test Data")
comp_a = Component(np.random.uniform(size=500))
comp_b = Component(np.random.normal(size=500))
data.add_component(comp_a, 'uniform')
data.add_component(comp_b, 'normal')
return data
def test_to_ccddata_invalid():
data = Data(label='not-an-image')
data.add_component(Component(np.array([3.4, 2.3, -1.1, 0.3]), units='Jy'),
'x')
with pytest.raises(ValueError) as exc:
data.get_object(CCDData, attribute=data.id['x'])
assert exc.value.args[
0] == 'Only 2-dimensional datasets can be converted to CCDData'
class FakeCoordinates(Coordinates):
def pixel_to_world_values(self, *pixel):
raise NotImplementedError()
def world_to_pixel_values(self, *pixel):
raise NotImplementedError()
coords = FakeCoordinates(n_dim=2)
coords.low_level_wcs = coords
data = Data(label='image-with-custom-coords', coords=coords)
data.add_component(Component(np.array([[3, 4], [4, 5]]), units='Jy'), 'x')
with pytest.raises(TypeError) as exc:
data.get_object(CCDData, attribute=data.id['x'])
assert exc.value.args[
0] == 'data.coords should be an instance of Coordinates or WCS'
def test_image():
data = Data(label="Test Image")
comp_a = Component(np.ones((25, 25)))
data.add_component(comp_a, 'test_1')
comp_b = Component(np.zeros((25, 25)))
data.add_component(comp_b, 'test_2')
return data
def setup_method(self, method):
# Set up simple spectral WCS
wcs_1d = WCS(naxis=1)
wcs_1d.wcs.ctype = ['VELO-LSR']
wcs_1d.wcs.set()
# Set up a spectral cube WCS
wcs_3d = WCS(naxis=3)
wcs_3d.wcs.ctype = ['RA---TAN', 'DEC--TAN', 'VELO-LSR']
wcs_3d.wcs.set()
# Set up glue Coordinates object
coords_1d = WCSCoordinates(wcs=wcs_1d)
coords_3d = WCSCoordinates(wcs=wcs_3d)
self.data_1d = Data(label='spectrum', coords=coords_1d)
self.data_3d = Data(label='spectrum', coords=coords_3d)
# FIXME: there should be an easier way to do this in glue
x = np.array([3.4, 2.3, -1.1, 0.3])
y = np.array([3.2, 3.3, 3.4, 3.5])
self.data_1d.add_component(Component(x, units='Jy'), 'x')
self.data_1d.add_component(Component(y, units='Jy'), 'y')
self.data_3d.add_component(Component(np.broadcast_to(x, (6, 5, 4)), units='Jy'), 'x')
self.data_3d.add_component(Component(np.broadcast_to(x, (6, 5, 4))), 'y')
self.app = GlueApplication()
self.session = self.app.session
self.hub = self.session.hub
self.data_collection = self.session.data_collection
self.data_collection.append(self.data_1d)
self.data_collection.append(self.data_3d)
def test_data():
data = Data(label="Test Data 1")
data2 = Data(label="Teset Data 2")
comp_a = Component(np.array([1, 2, 3]))
comp_b = Component(np.array([1, 2, 3]))
comp_c = Component(np.array([2, 4, 6]))
comp_d = Component(np.array([1, 3, 5]))
data.add_component(comp_a, 'a')
data.add_component(comp_b, 'b')
data2.add_component(comp_c, 'c')
data2.add_component(comp_d, 'd')
return data, data2
def test_limits_inf(self):
d = Data()
x = Component(np.array([[1, 2], [np.infty, 4]]))
y = Component(np.array([[2, 4], [-np.infty, 8]]))
xid = d.add_component(x, 'x')
yid = d.add_component(y, 'y')
self.collect.append(d)
self.client.add_layer(d)
self.client.xatt = xid
self.client.yatt = yid
assert self.client._visible_limits(0) == (1, 4)
assert self.client._visible_limits(1) == (2, 8)
def make_test_data():
data = Data(label="Test Cat Data 1")
comp_x1 = Component(np.array([4, 5, 6, 3]))
comp_y1 = Component(np.array([1, 2, 3, 2]))
comp_z1 = Component(np.array([2, 3, 4, 1]))
data.add_component(comp_x1, 'x_gal')
data.add_component(comp_y1, 'y_gal')
data.add_component(comp_z1, 'z_gal')
return data
def make_test_data():
data = Data(label="Test Cube Data")
np.random.seed(12345)
for letter in 'abc':
comp = Component(np.random.random((10, 10, 10)))
data.add_component(comp, letter)
# make sure one component key is primary
data.add_component(Component(np.random.random((10, 10, 10))), 'PRIMARY')
return data
def test_categorical_data():
data = Data(label="Test Cat Data 1")
data2 = Data(label="Teset Cat Data 2")
comp_x1 = CategoricalComponent(np.array(['a', 'a', 'b']))
comp_y1 = Component(np.array([1, 2, 3]))
comp_x2 = CategoricalComponent(np.array(['c', 'a', 'b']))
comp_y2 = Component(np.array([1, 3, 5]))
data.add_component(comp_x1, 'x1')
data.add_component(comp_y1, 'y1')
data2.add_component(comp_x2, 'x2')
data2.add_component(comp_y2, 'y2')
return data, data2
def test_to_spectrum1d():
# Set up simple spectral WCS
wcs = WCS(naxis=1)
wcs.wcs.ctype = ['VELO-LSR']
wcs.wcs.set()
coords = WCSCoordinates(wcs=wcs)
data = Data(label='spectrum', coords=coords)
data.add_component(Component(np.array([3.4, 2.3, -1.1, 0.3]), units='Jy'),
'x')
spec = data.get_object(Spectrum1D, attribute=data.id['x'])
assert_quantity_allclose(spec.spectral_axis, [1, 2, 3, 4] * u.m / u.s)
assert_quantity_allclose(spec.flux, [3.4, 2.3, -1.1, 0.3] * u.Jy)
data.add_subset(data.id['x'] > 1, label='bright')
spec_subset = data.get_subset_object(cls=Spectrum1D,
subset_id=0,
attribute=data.id['x'])
assert_quantity_allclose(spec_subset.spectral_axis,
[1, 2, 3, 4] * u.m / u.s)
assert_quantity_allclose(spec_subset.flux,
[3.4, 2.3, np.nan, np.nan] * u.Jy)
assert_equal(spec_subset.mask, [1, 1, 0, 0])
def test_to_ccddata(with_wcs):
if with_wcs:
coords = WCS_CELESTIAL
else:
coords = None
data = Data(label='image', coords=coords)
data.add_component(
Component(np.array([[3.4, 2.3], [-1.1, 0.3]]), units='Jy'), 'x')
image = data.get_object(CCDData, attribute=data.id['x'])
assert image.wcs is (WCS_CELESTIAL if with_wcs else None)
assert_allclose(image.data, [[3.4, 2.3], [-1.1, 0.3]])
assert image.unit is u.Jy
data.add_subset(data.id['x'] > 1, label='bright')
image_subset = data.get_subset_object(cls=CCDData,
subset_id=0,
attribute=data.id['x'])
assert image_subset.wcs is (WCS_CELESTIAL if with_wcs else None)
assert_allclose(image_subset.data, [[3.4, 2.3], [-1.1, 0.3]])
assert image_subset.unit is u.Jy
assert_equal(image_subset.mask, [[0, 0], [1, 1]])
def ds9_region(filename):
reg = regions.read_ds9(filename)
comp = Component(np.ones(len(reg), dtype='bool'))
data = RegionData(label='Regions: {0}'.format(os.path.split(filename)[-1]),
regions=reg)
return data
def load_stacked_sequence(self, raster_data):
for window, window_data in raster_data.items():
w_data = Data(label=f"{window.replace(' ', '_')}")
w_data.coords = window_data.wcs
w_data.add_component(Component(window_data.data),
f"{window.replace(' ', '_')}")
w_data.style = VisualAttributes(color='#7A617C')
self.datasets.append(w_data)
def test_to_spectral_cube_invalid_ndim():
data = Data(label='not-a-spectral-cube')
data.add_component(Component(np.array([3.4, 2.3, -1.1, 0.3]), units='Jy'), 'x')
with pytest.raises(ValueError) as exc:
data.get_object(SpectralCube, attribute=data.id['x'])
assert exc.value.args[0] == ('Data object should have 3 or 4 dimensions in order '
'to be converted to a SpectralCube object.')
def test_to_spectral_cube_missing_wcs():
data = Data(label='not-a-spectral-cube')
values = np.random.random((4, 5, 3))
data.add_component(Component(values, units='Jy'), 'x')
with pytest.raises(TypeError) as exc:
data.get_object(SpectralCube, attribute=data.id['x'])
assert exc.value.args[0] == ('data.coords should be an instance of BaseLowLevelWCS.')
def test_to_ccddata_unitless():
data = Data(label='image', coords=WCS_CELESTIAL)
data.add_component(Component(np.array([[3.4, 2.3], [-1.1, 0.3]])), 'x')
image = data.get_object(CCDData, attribute=data.id['x'])
assert_allclose(image.data, [[3.4, 2.3], [-1.1, 0.3]])
assert image.unit is u.one
def test_to_spectrum1d_invalid():
data = Data(label='not-a-spectrum')
data.add_component(Component(np.array([3.4, 2.3, -1.1, 0.3]), units='Jy'), 'x')
with pytest.raises(TypeError) as exc:
data.get_object(Spectrum1D, attribute=data.id['x'])
assert exc.value.args[0] == ('data.coords should be an instance of WCS '
'or SpectralCoordinates')
def test_ticks_go_back_after_changing(self):
""" If you change to a categorical axis and then change back
to a numeric, the axis ticks should fix themselves properly.
"""
data = Data()
data.add_component(Component(np.arange(100)), 'y')
data.add_component(CategoricalComponent(['a'] * 50 + ['b'] * 50),
'xcat')
data.add_component(Component(2 * np.arange(100)), 'xcont')
self.add_data(data=data)
self.client.yatt = data.find_component_id('y')
self.client.xatt = data.find_component_id('xcat')
self.check_ticks(self.client.axes.xaxis, False, True)
self.check_ticks(self.client.axes.yaxis, False, False)
self.client.xatt = data.find_component_id('xcont')
self.check_ticks(self.client.axes.yaxis, False, False)
self.check_ticks(self.client.axes.xaxis, False, False)
def test_to_spectral_cube_unitless(spectral_cube_wcs):
data = Data(label='spectral_cube', coords=spectral_cube_wcs)
values = np.random.random((4, 5, 3))
data.add_component(Component(values), 'x')
spec = data.get_object(SpectralCube, attribute=data.id['x'])
assert_quantity_allclose(spec.spectral_axis, [1, 2, 3, 4] * u.m / u.s)
assert_quantity_allclose(spec.filled_data[...], values * u.one)
def make_test_data():
data = Data(label="Test Cat Data 1")
np.random.seed(12345)
for letter in 'abcdefxyz':
comp = Component(np.random.random(100))
data.add_component(comp, letter)
return data
def make_test_data():
data = Data(label="Test Cube Data")
np.random.seed(12345)
for letter in 'abc':
comp = Component(np.random.random((10, 10, 10)))
data.add_component(comp, letter)
return data
def test_to_spectral_cube_invalid_wcs():
wcs = WCS(naxis=3)
data = Data(label='not-a-spectral-cube', coords=wcs)
values = np.random.random((4, 5, 3))
data.add_component(Component(values, units='Jy'), 'x')
with pytest.raises(ValueError) as exc:
data.get_object(SpectralCube, attribute=data.id['x'])
assert exc.value.args[0] == ('No celestial axes found in WCS')
wcs.wcs.ctype = ['RA---TAN', 'DEC--TAN', '']
data = Data(label='not-a-spectral-cube', coords=wcs)
values = np.random.random((4, 5, 3))
data.add_component(Component(values, units='Jy'), 'x')
with pytest.raises(ValueError) as exc:
data.get_object(SpectralCube, attribute=data.id['x'])
assert exc.value.args[0] == ('No spectral axes found in WCS')
def load_sunpy_map(self, sunpy_map):
sunpy_map_loaded = sunpy.map.Map(sunpy_map)
label = 'sunpy-map-' + sunpy_map_loaded.name
data = Data(label=label)
data.coords = sunpy_map_loaded.wcs # preferred way, preserves more info in some cases
data.meta = sunpy_map_loaded.meta
data.add_component(Component(sunpy_map_loaded.data),
sunpy_map_loaded.name)
data.style = VisualAttributes(color='#FDB813',
preferred_cmap=sunpy_map.cmap)
self.datasets.append(data)
def test_to_spectrum1d_with_spectral_coordinates():
coords = SpectralCoordinates([1, 4, 10] * u.micron)
data = Data(label='spectrum1d', coords=coords)
data.add_component(Component(np.array([3, 4, 5]), units='Jy'), 'x')
assert_allclose(data.coords.pixel2world([0, 0.5, 1, 1.5, 2]),
[[1, 2.5, 4, 7, 10]])
spec = data.get_object(Spectrum1D, attribute=data.id['x'])
assert_quantity_allclose(spec.spectral_axis, [1, 4, 10] * u.micron)
assert_quantity_allclose(spec.flux, [3, 4, 5] * u.Jy)
def test_to_spectral_cube_default_attribute(spectral_cube_wcs):
data = Data(label='spectral_cube', coords=spectral_cube_wcs)
values = np.random.random((4, 5, 3))
with pytest.raises(ValueError) as exc:
data.get_object(SpectralCube)
assert exc.value.args[0] == 'Data object has no attributes.'
data.add_component(Component(values, units='Jy'), 'x')
spec = data.get_object(SpectralCube)
assert_quantity_allclose(spec.filled_data[...], values * u.Jy)
data.add_component(Component(values, units='Jy'), 'y')
with pytest.raises(ValueError) as exc:
data.get_object(SpectralCube)
assert exc.value.args[0] == ('Data object has more than one attribute, so '
'you will need to specify which one to use as '
'the flux for the spectral cube using the attribute= '
'keyword argument.')
def test_to_ccddata_default_attribute():
data = Data(label='image', coords=WCS_CELESTIAL)
with pytest.raises(ValueError) as exc:
data.get_object(CCDData)
assert exc.value.args[0] == 'Data object has no attributes.'
data.add_component(Component(np.array([[3, 4], [5, 6]]), units='Jy'), 'x')
image = data.get_object(CCDData)
assert_allclose(image.data, [[3, 4], [5, 6]])
assert image.unit is u.Jy
data.add_component(Component(np.array([[3, 4], [5, 6]]), units='Jy'), 'y')
with pytest.raises(ValueError) as exc:
data.get_object(CCDData)
assert exc.value.args[0] == ('Data object has more than one attribute, so '
'you will need to specify which one to use as '
'the flux for the spectrum using the attribute= '
'keyword argument.')
def load_sji(self, sji):
with fits.open(sji) as hdul:
hdul.verify("fix")
label = hdul[0].header['TDESC1'] + hdul[0].header['OBSID']
data = Data(label=label)
data.coords = WCSCoordinates(hdul[0].header)
data.meta = hdul[0].header
preferred_cmap_name = 'IRIS ' + hdul[0].header['TDESC1'].replace(
'_', ' ')
data.style = VisualAttributes(preferred_cmap=preferred_cmap_name)
data.add_component(Component(hdul[0].data), label)
self.datasets.append(data)
def load_sequence(self, raster_data):
for window, window_data in raster_data.items():
for i, scan_data in enumerate(window_data):
w_data = Data(
label=
f"{window.replace(' ', '_')}-{scan_data.meta['OBSID']}-scan-{i}"
)
w_data.coords = scan_data.wcs
w_data.add_component(Component(scan_data.data),
f"{window.replace(' ', '_')}-scan-{i}")
w_data.meta = scan_data.meta
w_data.style = VisualAttributes(color='#5A4FCF')
self.datasets.append(w_data)
def test_high_cardinatility_timing(self):
card = 50000
data = Data()
card_data = [str(num) for num in range(card)]
data.add_component(Component(np.arange(card * 5)), 'y')
data.add_component(CategoricalComponent(np.repeat([card_data], 5)),
'xcat')
self.add_data(data)
comp = data.find_component_id('xcat')
timer_func = partial(self.client._set_xydata, 'x', comp)
timer = timeit(timer_func, number=1)
assert timer < 3 # this is set for Travis speed
def panda_process(indf):
"""
Build a data set from a table using pandas. This attempts to respect
categorical data input by letting pandas.read_csv infer the type
"""
result = Data()
for name, column in indf.iteritems():
if (column.dtype == np.object) | (column.dtype == np.bool):
# try to salvage numerical data
try:
coerced = pd.to_numeric(column, errors='coerce')
except AttributeError: # pandas < 0.19
coerced = column.convert_objects(convert_numeric=True)
if (coerced.dtype != column.dtype) and coerced.isnull().mean() < 0.4:
c = Component(coerced.values)
else:
# pandas has a 'special' nan implementation and this doesn't
# play well with np.unique
c = CategoricalComponent(column.fillna(''))
else:
c = Component.autotyped(column.values)
# convert header to string - in some cases if the first row contains
# numbers, these are cast to numerical types, so we want to change that
# here.
if not isinstance(name, six.string_types):
name = str(name)
# strip off leading #
name = name.strip()
if name.startswith('#'):
name = name[1:].strip()
result.add_component(c, name)
return result
def add_component(self, component, label, hidden=False):
""" Add a new component to this data set.
:param component: object to add. Can be a Component,
array-like object, or ComponentLink
:param label:
The label. If this is a string,
a new :class:`glue.core.component_id.ComponentID` with this label will be
created and associated with the Component
:type component: :class:`~glue.core.component.Component` or
array-like
:type label: :class:`str` or :class:`~glue.core.component_id.ComponentID`
:raises:
TypeError, if label is invalid
ValueError if the component has an incompatible shape
:returns:
The ComponentID associated with the newly-added component
"""
if isinstance(component, ComponentLink):
return self.add_component_link(component, label=label, hidden=hidden)
if not isinstance(component, Component):
component = Component.autotyped(component)
if isinstance(component, DerivedComponent):
if len(self._components) == 0:
raise TypeError("Cannot add a derived component as a first component")
component.set_parent(self)
if not(self._check_can_add(component)):
raise ValueError("The dimensions of component %s are "
"incompatible with the dimensions of this data: "
"%r vs %r" % (label, component.shape, self.shape))
if isinstance(label, ComponentID):
component_id = label
if component_id.parent is None:
component_id.parent = self
else:
component_id = ComponentID(label, hidden=hidden, parent=self)
if len(self._components) == 0:
self._create_pixel_and_world_components(ndim=component.ndim)
# In some cases, such as when loading a session, we actually disable the
# auto-creation of pixel and world coordinates, so the first component
# may be a coordinate component with no shape. Therefore we only set the
# shape once a component has a valid shape rather than strictly on the
# first component.
if self._shape == () and component.shape != ():
self._shape = component.shape
is_present = component_id in self._components
self._components[component_id] = component
if self.hub and not is_present:
msg = DataAddComponentMessage(self, component_id)
self.hub.broadcast(msg)
msg = ComponentsChangedMessage(self)
self.hub.broadcast(msg)
return component_id
def add_component(self, component, label, hidden=False):
""" Add a new component to this data set.
:param component: object to add. Can be a Component,
array-like object, or ComponentLink
:param label:
The label. If this is a string,
a new :class:`glue.core.component_id.ComponentID` with this label will be
created and associated with the Component
:type component: :class:`~glue.core.component.Component` or
array-like
:type label: :class:`str` or :class:`~glue.core.component_id.ComponentID`
:raises:
TypeError, if label is invalid
ValueError if the component has an incompatible shape
:returns:
The ComponentID associated with the newly-added component
"""
if isinstance(component, ComponentLink):
component = DerivedComponent(self, component)
if not isinstance(component, Component):
component = Component.autotyped(component)
if isinstance(component, DerivedComponent):
component.set_parent(self)
if not(self._check_can_add(component)):
raise ValueError("The dimensions of component %s are "
"incompatible with the dimensions of this data: "
"%r vs %r" % (label, component.shape, self.shape))
if isinstance(label, ComponentID):
component_id = label
else:
component_id = ComponentID(label, hidden=hidden)
is_present = component_id in self._components
self._components[component_id] = component
first_component = len(self._components) == 1
if first_component:
if isinstance(component, DerivedComponent):
raise TypeError("Cannot add a derived component as "
"first component")
self._shape = component.shape
self._create_pixel_and_world_components()
if self.hub and (not is_present):
msg = DataAddComponentMessage(self, component_id)
self.hub.broadcast(msg)
msg = ComponentsChangedMessage(self)
self.hub.broadcast(msg)
return component_id
