def _create_empty_events_data_array(
tof_dtype: Any = np.int64,
tof_unit: Union[str, sc.Unit] = "ns",
detector_id_dtype: Any = np.int32) -> sc.DataArray:
data = sc.DataArray(data=sc.empty(dims=[_event_dimension],
shape=[0],
unit='counts',
with_variances=True,
dtype=np.float32),
coords={
_time_of_flight:
sc.empty(dims=[_event_dimension],
shape=[0],
dtype=tof_dtype,
unit=tof_unit),
_detector_dimension:
sc.empty(dims=[_event_dimension],
shape=[0],
dtype=detector_id_dtype),
})
indices = sc.array(dims=[_pulse_dimension], values=[], dtype='int64')
return sc.DataArray(data=sc.bins(begin=indices,
end=indices,
dim=_event_dimension,
data=data),
coords={
'pulse_time':
sc.zeros(dims=[_pulse_dimension],
shape=[0],
dtype='datetime64',
unit='ns')
})
def test_dataset_histogram():
var = sc.Variable(dims=['x'], shape=[2], dtype=sc.dtype.event_list_float64)
var['x', 0].values = np.arange(3)
var['x', 0].values.append(42)
var['x', 0].values.extend(np.ones(3))
var['x', 1].values = np.ones(6)
ds = sc.Dataset()
s = sc.DataArray(data=sc.Variable(dims=['x'],
values=np.ones(2),
variances=np.ones(2)),
coords={'y': var})
s1 = sc.DataArray(data=sc.Variable(dims=['x'],
values=np.ones(2),
variances=np.ones(2)),
coords={'y': var * 5.0})
realign_coords = {
'y': sc.Variable(values=np.arange(5, dtype=np.float64), dims=['y'])
}
ds['s'] = sc.realign(s, realign_coords)
ds['s1'] = sc.realign(s1, realign_coords)
h = sc.histogram(ds)
assert np.array_equal(
h['s'].values, np.array([[1.0, 4.0, 1.0, 0.0], [0.0, 6.0, 0.0, 0.0]]))
assert np.array_equal(
h['s1'].values, np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]))
def test_slice_init():
orig = sc.DataArray(
data=sc.Variable(['x'], values=np.arange(2.0)),
coords={'x': sc.Variable(['x'], values=np.arange(3.0))})
a = sc.DataArray(orig['x', :])
assert a == orig
b = sc.DataArray(orig['x', 1:])
assert b.data.values[0] == orig.data.values[1:]
def test_filter():
assert_export(sc.filter,
data=sc.DataArray(1.0 * sc.units.m),
filter="x",
interval=sc.Variable())
assert_export(sc.filter,
data=sc.DataArray(1.0 * sc.units.m),
filter="x",
interval=sc.Variable(),
keep_attrs=False)
def test_bins_arithmetic():
var = sc.Variable(dims=['event'], values=[1.0, 2.0, 3.0, 4.0])
table = sc.DataArray(var, {'x': var})
binned = sc.bin(table, [sc.Variable(dims=['x'], values=[1.0, 5.0])])
hist = sc.DataArray(
data=sc.Variable(dims=['x'], values=[1.0, 2.0]),
coords={'x': sc.Variable(dims=['x'], values=[1.0, 3.0, 5.0])})
binned.bins *= sc.lookup(func=hist, dim='x')
assert sc.is_equal(
binned.bins.data.data,
sc.Variable(dims=['event'], values=[1.0, 2.0, 6.0, 8.0]))
def test_load_component_info_to_2d_geometry(geom_file):
geometry = mantid.load_component_info_to_2d(geom_file,
sizes={
'x': 10,
'y': 10
})
assert geometry["position"].sizes == {'x': 10, 'y': 10}
assert sc.identical(
geometry["x"],
sc.DataArray(data=sc.array(
dims=["x"], values=np.arange(0.0, 0.1, 0.01), unit=sc.units.m)))
assert sc.identical(
geometry["y"],
sc.DataArray(data=sc.array(
dims=["y"], values=np.arange(0.0, 0.1, 0.01), unit=sc.units.m)))
def test_data_array_dtype_scipp_container():
a = sc.DataArray(data=x)
a.coords['scalar'] = sc.Variable(value=a)
a.coords['1d'] = sc.Variable(dims=a.dims,
shape=a.shape,
dtype=sc.dtype.DataArray)
a.coords['1d'].values[0] = sc.DataArray(data=0.0 * sc.units.m)
a.coords['1d'].values[1] = sc.DataArray(data=1.0 * sc.units.m)
a.coords['1d'].values[2] = sc.DataArray(data=2.0 * sc.units.m)
a.coords['1d'].values[3] = sc.DataArray(data=3.0 * sc.units.m)
a.coords['dataset'] = sc.Variable(value=sc.Dataset({
'a': array_1d,
'b': array_2d
}))
check_roundtrip(a)
def _getitem(self, select: ScippIndex) -> sc.DataArray:
dims = self.dims
index = to_plain_index(dims, select)
signal = self._loader.load_dataset(self._group,
self._signal_name,
dimensions=dims,
index=index)
if self._errors_name in self:
stddevs = self._loader.load_dataset(self._group,
self._errors_name,
dimensions=dims,
index=index)
signal.variances = sc.pow(stddevs, 2).values
da = sc.DataArray(data=signal)
if 'axes' in self.attrs:
# Unlike self.dims we *drop* entries that are '.'
coords = [a for a in self.attrs['axes'] if a != '.']
else:
coords = self._signal.attrs['axes'].split(',')
for dim in coords:
index = to_plain_index([dim], select, ignore_missing=True)
da.coords[dim] = self._loader.load_dataset(self._group,
dim,
dimensions=[dim],
index=index)
return da
def test_create_from_data_array_and_variable_mix():
var_1 = sc.Variable(dims=['x'], values=np.arange(4))
var_2 = sc.Variable(dims=['x'], values=np.arange(4))
da = sc.DataArray(data=var_1, coords={'x': var_1, 'aux': var_1})
d = sc.Dataset({'array': da, 'variable': var_2})
assert sc.is_equal(d['array'], da)
assert sc.is_equal(d['variable'].data, var_2)
def test_mean_all():
var = sc.Variable(['x', 'y'], values=np.arange(4.0).reshape(2, 2))
mask = sc.Variable(['x', 'y'],
values=np.array([[False, False], [True, False]]))
da = sc.DataArray(var, masks={'m': mask}) # Add masks
assert sc.sum(da).data.value == 0 + 1 + 3 # 2.0 masked
sc.mean(da).data.value == 4 / 3
def _append_translation(offset: np.ndarray, transform: GroupObject,
transformations: List[np.ndarray],
direction_unit_vector: np.ndarray, group_name: str,
nexus: LoadFromNexus):
loaded_transform = _get_transformation_magnitude_and_unit(
group_name, transform, nexus)
loaded_transform_m = loaded_transform.to(dtype=sc.DType.float64,
unit=sc.units.m,
copy=False)
# -1 as describes passive transformation
vectors = sc.vector(value=(offset -
direction_unit_vector)) * loaded_transform_m
translations = sc.spatial.translations(dims=loaded_transform_m.dims,
values=vectors.values,
unit=sc.units.m)
if isinstance(loaded_transform, sc.DataArray):
t = sc.DataArray(data=translations,
coords={"time": loaded_transform.coords["time"]})
else:
t = translations
transformations.append(t)
def test_create_from_data_arrays():
var1 = sc.Variable(dims=['x'], values=np.arange(4))
var2 = sc.Variable(dims=['x'], values=np.ones(4))
da1 = sc.DataArray(var1, coords={'x': var1, 'aux': var2})
da2 = sc.DataArray(var1, coords={'x': var1, 'aux': var2})
d = sc.Dataset()
d['a'] = da1
d['b'] = da2
assert d == sc.Dataset(data={
'a': var1,
'b': var1
},
coords={
'x': var1,
'aux': var2
})
def mean_from_adj_pixels(data):
"""
Applies a mean across 8 neighboring pixels (plus centre value)
for data with 'x' and 'y' dimensions (at least).
Result will calculate mean from slices across additional dimensions.
For example if there is a tof dimension in addition to x, and y,
for each set of neighbours the returned mean will take the mean
tof value in the neighbour group.
"""
fill = np.finfo(data.values.dtype).min
has_variances = data.variances is not None
container = sc.empty(dims=['neighbor'] + data.dims,
dtype=data.dtype,
shape=[
9,
] + data.shape,
with_variances=has_variances,
unit=data.unit)
container['neighbor', 0] = data
container['neighbor', 1] = _shift(data, "x", True, fill)
container['neighbor', 2] = _shift(data, "x", False, fill)
container['neighbor', 3] = _shift(data, "y", True, fill)
container['neighbor', 4] = _shift(data, "y", False, fill)
container['neighbor', 5:7] = _shift(container['neighbor', 1:3], "y", True,
fill)
container['neighbor', 7:9] = _shift(container['neighbor', 1:3], "y", False,
fill)
edges_mask = container <= sc.scalar(value=fill, unit=data.unit)
da = sc.DataArray(data=container, masks={'edges': edges_mask})
return sc.mean(da, dim='neighbor').data
def convert_from_str_unit_and_dtype(d):
delete_dtype = False
for k, v in d.items():
if isinstance(v, dict):
convert_from_str_unit_and_dtype(v)
if k not in ("attrs", "masks", "coords"):
if {"coords", "data"}.issubset(set(v.keys())):
# from_dict does not work with nested DataArrays,
# so we have to manually construct DataArrays here.
d[k] = sc.DataArray(coords=v["coords"],
data=v["data"],
attrs=v["attrs"])
else:
try:
if any(scipp_container_type in str(v["dtype"]) for
scipp_container_type in _scipp_containers):
del v["dtype"]
except KeyError:
pass
d[k] = sc.from_dict(v)
else:
if k == "dtype":
try:
d[k] = np.dtype(v)
except TypeError:
if v == "string":
d[k] = sc.DType.string
elif any(
scipp_container_type in k
for scipp_container_type in _scipp_containers):
delete_dtype = True
# Delete now, not while looping through dictionary
if delete_dtype:
del d["dtype"]
def _bin_events(data: DetectorData):
if not bin_by_pixel:
# If loading "raw" data, leave binned by pulse.
return data.event_data
if data.detector_ids is None:
# If detector ids were not found in an associated detector group
# we will just have to bin according to whatever
# ids we have a events for (pixels with no recorded events
# will not have a bin)
event_id = data.event_data.bins.constituents['data'].coords[
_detector_dimension]
data.detector_ids = sc.array(dims=[_detector_dimension],
values=np.unique(event_id.values))
# Events in the NeXus file are effectively binned by pulse
# (because they are recorded chronologically)
# but for reduction it is more useful to bin by detector id
# Broadcast pulse times to events
data.event_data.bins.coords['pulse_time'] = sc.bins_like(
data.event_data, fill_value=data.event_data.coords['pulse_time'])
# TODO Look into using `erase=[_pulse_dimension]` instead of binning
# underlying buffer. Must prove that performance can be unaffected.
da = sc.bin(data.event_data.bins.constituents['data'],
groups=[data.detector_ids])
# Add a single time-of-flight bin
da = sc.DataArray(data=sc.broadcast(da.data,
dims=da.dims + [_time_of_flight],
shape=da.shape + [1]),
coords={_detector_dimension: data.detector_ids})
if pixel_positions_loaded:
# TODO: the name 'position' should probably not be hard-coded but moved
# to a variable that cah be changed in a single place.
da.coords['position'] = data.pixel_positions
return da
def test_events_does_not_repeat_dense_coords():
events = sc.Variable(['y', 'z'], shape=(3, sc.Dimensions.Events))
events.values[0].extend(np.arange(0))
events.values[2].extend(np.arange(0))
events.values[1].extend(np.arange(0))
d = sc.Dataset()
d['a'] = sc.Variable(['y', 'x', 'z'], shape=(3, 2, 4), variances=True)
d['b'] = sc.DataArray(
events,
coords={
'y': sc.Variable(['y'], values=np.arange(4)),
'z': sc.Variable(['z'], shape=(sc.Dimensions.Events, )),
"binedge": sc.Variable(['y'], values=np.random.rand(4))
},
attrs={"attr": sc.Variable(['y'], values=np.random.rand(3))})
html = BeautifulSoup(make_html(d), features="html.parser")
sections = html.find_all(class_="xr-section-summary")
assert ["Coordinates" in section.text
for section in sections].count(True) == 2
attr_section = next(section for section in sections
if "Attributes" in section.text)
# check that this section is a subsection
assert "sc-subsection" in attr_section.parent.attrs["class"]
variables = html.find_all(class_="xr-var-name")
# check that each dim is only present once
assert ["z" in var.text for var in variables].count(True) == 1
assert ["y" in var.text for var in variables].count(True) == 1
def test_plot_2d_with_dimension_of_size_2():
a = sc.DataArray(data=sc.Variable(dims=['y', 'x'], shape=[2, 4]),
coords={
'x': sc.Variable(dims=['x'], values=[1, 2, 3, 4]),
'y': sc.Variable(dims=['y'], values=[1, 2])
})
plot(a)
def test_to_workspace_2d(param_dim):
from mantid.simpleapi import mtd
mtd.clear()
data_len = 2
expected_bins = data_len + 1
expected_number_spectra = 10
y = sc.Variable(dims=['spectrum', param_dim],
values=np.random.rand(expected_number_spectra, data_len),
variances=np.random.rand(expected_number_spectra,
data_len))
x = sc.Variable(dims=['spectrum', param_dim],
values=np.arange(expected_number_spectra * expected_bins,
dtype=np.float64).reshape(
(expected_number_spectra,
expected_bins)))
data = sc.DataArray(data=y, coords={param_dim: x})
ws = scn.to_mantid(data, param_dim)
assert len(ws.readX(0)) == expected_bins
assert ws.getNumberHistograms() == expected_number_spectra
# check that no workspaces have been leaked in the ADS
assert len(mtd) == 0, f"Workspaces present: {mtd.getObjectNames()}"
for i in range(expected_number_spectra):
np.testing.assert_array_equal(ws.readX(i), x['spectrum', i].values)
np.testing.assert_array_equal(ws.readY(i), y['spectrum', i].values)
np.testing.assert_array_equal(ws.readE(i),
np.sqrt(y['spectrum', i].variances))
def test_data_array_to_dict():
da = sc.DataArray(
coords={
"x": sc.Variable(dims=["x"], values=np.arange(10.)),
"y": sc.Variable(dims=["y"], values=np.arange(5), unit=sc.units.m),
},
masks={
"amask":
sc.Variable(dims=["y"], values=[True, True, False, True, False])
},
attrs={"attr1": sc.Variable(dims=["x"], values=np.random.random(10))},
data=sc.Variable(dims=["y", "x"], values=np.random.random([5, 10])))
da_dict = sc.to_dict(da)
assert da_dict["data"]["dims"] == da.dims
assert da_dict["data"]["shape"] == da.shape
assert np.array_equal(da_dict["data"]["values"], da.values)
assert np.array_equal(da_dict["data"]["variances"], da.variances)
assert da_dict["data"]["unit"] == da.unit
assert da_dict["data"]["dtype"] == da.dtype
assert sc.is_equal(sc.from_dict(da_dict["coords"]["x"]), da.coords["x"])
assert sc.is_equal(sc.from_dict(da_dict["coords"]["y"]), da.coords["y"])
assert sc.is_equal(sc.from_dict(da_dict["masks"]["amask"]),
da.masks["amask"])
assert sc.is_equal(sc.from_dict(da_dict["attrs"]["attr1"]),
da.attrs["attr1"])
def test_astype_bad_conversion():
a = sc.DataArray(data=sc.Variable(['x'],
values=np.arange(10.0, dtype=np.int64)),
coords={'x': sc.Variable(['x'], values=np.arange(10.0))})
assert a.dtype == sc.dtype.int64
with pytest.raises(RuntimeError):
a.astype(sc.dtype.string)
def test_astype():
a = sc.DataArray(data=sc.Variable(['x'],
values=np.arange(10.0, dtype=np.int64)),
coords={'x': sc.Variable(['x'], values=np.arange(10.0))})
assert a.dtype == sc.dtype.int64
a_as_float = a.astype(sc.dtype.float32)
assert a_as_float.dtype == sc.dtype.float32
def test_nansum_all():
da = sc.DataArray(sc.Variable(['x', 'y'], values=np.ones(10).reshape(5,
2)))
da.data.values[0, 0] = np.nan
ds = sc.Dataset({'a': da})
assert np.isnan(sc.sum(da).data.value) # sanity check
assert sc.is_equal(sc.nansum(da).data, sc.Variable(value=9.0))
assert sc.is_equal(sc.nansum(da), sc.nansum(ds)['a'])
def test_plot_2d_with_decreasing_edges():
a = sc.DataArray(data=sc.Variable(dims=['y', 'x'],
values=np.arange(12).reshape(3, 4)),
coords={
'x': sc.Variable(dims=['x'], values=[4, 3, 2, 1]),
'y': sc.Variable(dims=['y'], values=[1, 2, 3])
})
plot(a)
def test_nanmean_all():
var = sc.Variable(['x', 'y'], values=np.arange(4.0).reshape(2, 2))
var['x', 0]['y', 1].value = np.nan
mask = sc.Variable(['x', 'y'],
values=np.array([[False, False], [True, False]]))
da = sc.DataArray(var, masks={'m': mask}) # Add masks
assert sc.nansum(da).data.value == 0 + 3 # 2.0 masked, 1.0 is nan
sc.mean(da).data.value == 3 / 2
def _table_to_data_array(table, key, value, stddev):
stddevs = table[stddev].values
dim = 'parameter'
coord = table[key].data.copy().rename_dims({'row': dim})
return sc.DataArray(data=sc.Variable(dims=[dim],
values=table[value].values,
variances=stddevs * stddevs),
coords={dim: coord})
def test_round_trip_dataarray():
data = sc.Variable(['tof'], values=[10., 10.], unit=sc.units.counts)
tof = sc.Variable(['tof'], values=[1., 2., 3.], unit=sc.units.us)
d = sc.DataArray(data, coords={'tof': tof})
dd = sc.density_to_counts(sc.counts_to_density(d, 'tof'), 'tof')
assert sc.is_equal(dd, d)
def test_round_trip_dataarray():
data = sc.Variable([Dim.Tof], values=[10., 10.], unit=sc.units.counts)
tof = sc.Variable([Dim.Tof], values=[1., 2., 3.], unit=sc.units.us)
d = sc.DataArray(data, coords={Dim.Tof: tof})
dd = sc.density_to_counts(sc.counts_to_density(d, Dim.Tof), Dim.Tof)
assert dd == d
def test_create_events_via_DataArray():
d = sc.Dataset()
events = sc.Variable(dims=[], shape=[], dtype=sc.dtype.event_list_float64)
d['a'] = sc.DataArray(data=events, coords={'x': events})
assert len(d.coords) == 1
assert len(d['a'].coords) == 1
assert d['a'].coords['x'] == events
assert d['a'].coords['x'] == d.coords['x']
def test_sparse_data_slice_with_on_the_fly_histogram():
N = 50
M = 10
var = sc.Variable(dims=[Dim.X, Dim.Tof],
shape=[M, sc.Dimensions.Sparse],
unit=sc.units.us)
for i in range(M):
v = np.random.normal(50.0, scale=20.0, size=int(np.random.rand() * N))
var[Dim.X, i].values = v
d = sc.Dataset()
d.coords[Dim.X] = sc.Variable([Dim.X],
values=np.arange(M),
unit=sc.units.m)
d['a'] = sc.DataArray(coords={Dim.Tof: var})
d['b'] = sc.DataArray(coords={Dim.Tof: var * 1.1})
plot(d[Dim.X, 4], bins=100)
def test_dataset_data_access():
var = sc.Variable(dims=['x'], shape=[2], dtype=sc.dtype.event_list_float64)
ds = sc.Dataset()
ds['events'] = sc.DataArray(data=sc.Variable(dims=['x'],
values=np.ones(2),
variances=np.ones(2)),
coords={'y': var})
assert ds['events'].values is not None
