def _create_empty_events_data_array(
tof_dtype: Any = np.int64,
tof_unit: Union[str, sc.Unit] = "ns",
detector_id_dtype: Any = np.int32) -> sc.DataArray:
data = sc.DataArray(data=sc.empty(dims=[_event_dimension],
shape=[0],
unit='counts',
with_variances=True,
dtype=np.float32),
coords={
_time_of_flight:
sc.empty(dims=[_event_dimension],
shape=[0],
dtype=tof_dtype,
unit=tof_unit),
_detector_dimension:
sc.empty(dims=[_event_dimension],
shape=[0],
dtype=detector_id_dtype),
})
indices = sc.array(dims=[_pulse_dimension], values=[], dtype='int64')
return sc.DataArray(data=sc.bins(begin=indices,
end=indices,
dim=_event_dimension,
data=data),
coords={
'pulse_time':
sc.zeros(dims=[_pulse_dimension],
shape=[0],
dtype='datetime64',
unit='ns')
})
def test_bins_default_begin_end():
data = sc.Variable(dims=['x'], values=[1, 2, 3, 4])
var = sc.bins(dim='x', data=data)
assert var.dims == data.dims
assert var.shape == data.shape
for i in range(4):
assert sc.is_equal(var['x', i].value, data['x', i])
def test_bins_default_end():
data = sc.Variable(dims=['x'], values=[1, 2, 3, 4])
begin = sc.Variable(dims=['y'], values=[1, 3], dtype=sc.dtype.int64)
var = sc.bins(begin=begin, dim='x', data=data)
assert var.dims == begin.dims
assert var.shape == begin.shape
assert sc.is_equal(var['y', 0].value, data['x', 1])
assert sc.is_equal(var['y', 1].value, data['x', 3])
def test_bins():
data = sc.Variable(dims=['x'], values=[1, 2, 3, 4])
begin = sc.Variable(dims=['y'], values=[0, 2], dtype=sc.dtype.int64)
end = sc.Variable(dims=['y'], values=[2, 4], dtype=sc.dtype.int64)
var = sc.bins(begin=begin, end=end, dim='x', data=data)
assert var.dims == begin.dims
assert var.shape == begin.shape
assert sc.is_equal(var['y', 0].value, data['x', 0:2])
assert sc.is_equal(var['y', 1].value, data['x', 2:4])
def make_tof_binned_events():
buffer = sc.DataArray(sc.zeros(dims=['event'], shape=[7], dtype=float),
coords={
'tof':
sc.array(dims=['event'],
values=[
1000.0, 3000.0, 2000.0, 4000.0,
5000.0, 6000.0, 3000.0
],
unit='us')
})
return sc.bins(data=buffer,
dim='event',
begin=sc.array(dims=['spectrum'],
values=[0, 4],
dtype='int64'),
end=sc.array(dims=['spectrum'],
values=[4, 7],
dtype='int64'))
def test_load_events_bins():
with in_memory_nexus_file_with_event_data() as nexus_file:
event_data_groups = find_by_nx_class("NXevent_data", nexus_file)
# Load only the first event data group we found
event_group = event_data_groups[0]
event_index_np = event_group["event_index"][...]
event_time_offset = sc.Variable(
['event'], values=event_group["event_time_offset"][...])
event_id = sc.Variable(['event'], values=event_group["event_id"][...])
event_index = sc.Variable(['pulse'],
values=event_index_np,
dtype=np.int64)
event_time_zero = sc.Variable(
['pulse'], values=event_group["event_time_zero"][...])
# Calculate the end index for each pulse
# The end index for a pulse is the start index of the next pulse
end_indices = event_index_np.astype(np.int64)
end_indices = np.roll(end_indices, -1)
end_indices[-1] = event_id.shape[0]
end_indices = sc.Variable(['pulse'], values=end_indices, dtype=np.int64)
# Weights are not stored in NeXus, so use 1s
weights = sc.Variable(['event'],
values=np.ones(event_id.shape),
dtype=np.float32)
data = sc.DataArray(data=weights,
coords={
'tof': event_time_offset,
'detector-id': event_id
})
events = sc.DataArray(data=sc.bins(begin=event_index,
end=end_indices,
dim='event',
data=data),
coords={'pulse-time': event_time_zero})
assert events.dims == event_index.dims
assert events.shape == event_index.shape
assert sc.is_equal(events['pulse', 0].value.coords['detector-id'],
data.coords['detector-id']['event', 0:3])
def test_convert_binned_events_converted(target):
tof = make_test_data(coords=('Ltotal', 'two_theta'), dataset=True)
del tof['counts']
tof['events'] = make_tof_binned_events()
# Construct events with all coords.
binned_tof = tof['events'].copy()
for name in ('Ltotal', 'two_theta'):
binned_tof.bins.coords[name] = sc.bins_like(binned_tof,
tof.coords[name])
dense_tof = binned_tof.bins.constituents['data']
expected = scn.convert(dense_tof,
origin='tof',
target=target,
scatter=True)
for intermediate in ('Ltotal', 'two_theta'):
expected.attrs.pop(intermediate, None)
expected.coords.pop(intermediate, None)
expected = sc.bins(**{**binned_tof.bins.constituents, 'data': expected})
res = scn.convert(tof, origin='tof', target=target, scatter=True)
assert sc.identical(res['events'].data, expected)
def _create_empty_event_data(detectors: List[DetectorData]):
"""
If any NXdetector groups had pixel position data but no events
then add an empty data array to make it easier to concatenate
the data from different groups
"""
empty_events = None
detector_id_dtype = None
for data in detectors:
if data.event_data is not None:
# If any event data were loaded then use an empty data
# array with the same data types
constituents = data.event_data.bins.constituents
constituents['begin'] = sc.zeros_like(constituents['begin'])
constituents['end'] = sc.zeros_like(constituents['end'])
constituents['data'] = constituents['data'][_event_dimension,
0:0].copy()
empty_events = data.event_data.copy(deep=False)
empty_events.data = sc.bins(**constituents)
break
elif data.detector_ids is not None:
detector_id_dtype = data.detector_ids.dtype
if empty_events is None:
if detector_id_dtype is None:
# Create empty data array with types/unit matching streamed event
# data, this avoids need to convert to concatenate with event data
# arriving from stream
empty_events = _create_empty_events_data_array(
np.int64, "ns", np.int32)
else:
# If detector_ids were loaded then match the type used for those
empty_events = _create_empty_events_data_array(
np.int64, "ns", detector_id_dtype)
for data in detectors:
if data.event_data is None:
data.event_data = empty_events
def _load_event_group(group: Group,
nexus: LoadFromNexus,
quiet: bool,
select=tuple()) -> DetectorData:
_check_for_missing_fields(group, nexus)
index = to_plain_index([_pulse_dimension], select)
def shape(name):
return nexus.get_shape(nexus.get_dataset_from_group(group, name))
max_index = shape("event_index")[0]
single = False
if index is Ellipsis or index == tuple():
last_loaded = False
else:
if isinstance(index, int):
single = True
start, stop, _ = slice(index, None).indices(max_index)
if start == stop:
raise IndexError('Index {start} is out of range')
index = slice(start, start + 1)
start, stop, stride = index.indices(max_index)
if stop + stride > max_index:
last_loaded = False
else:
stop += stride
last_loaded = True
index = slice(start, stop, stride)
event_index = nexus.load_dataset_from_group_as_numpy_array(
group, "event_index", index)
event_time_zero = _load_event_time_zero(group, nexus, index)
num_event = shape("event_time_offset")[0]
# Some files contain uint64 "max" indices, which turn into negatives during
# conversion to int64. This is a hack to get arround this.
event_index[event_index < 0] = num_event
if len(event_index) > 0:
event_select = slice(event_index[0],
event_index[-1] if last_loaded else num_event)
else:
event_select = slice(None)
if nexus.dataset_in_group(group, "event_id")[0]:
event_id = nexus.load_dataset(group,
"event_id", [_event_dimension],
index=event_select)
else:
event_id = None
event_time_offset = nexus.load_dataset(group,
"event_time_offset",
[_event_dimension],
index=event_select)
# Weights are not stored in NeXus, so use 1s
weights = sc.ones(dims=[_event_dimension],
shape=event_time_offset.shape,
unit='counts',
dtype=np.float32,
with_variances=True)
events = sc.DataArray(data=weights,
coords={'event_time_offset': event_time_offset})
if event_id is not None:
events.coords['event_id'] = event_id
if not last_loaded:
event_index = np.append(event_index, num_event)
else:
# Not a bin-edge coord, all events in bin are associated with same (previous)
# pulse time value
event_time_zero = event_time_zero[:-1]
event_index = sc.array(dims=[_pulse_dimension],
values=event_index,
dtype=sc.DType.int64)
event_index -= event_index.min()
# There is some variation in the last recorded event_index in files from different
# institutions. We try to make sure here that it is what would be the first index of
# the next pulse. In other words, ensure that event_index includes the bin edge for
# the last pulse.
if single:
begins = event_index[_pulse_dimension, 0]
ends = event_index[_pulse_dimension, 1]
event_time_zero = event_time_zero[_pulse_dimension, 0]
else:
begins = event_index[_pulse_dimension, :-1]
ends = event_index[_pulse_dimension, 1:]
try:
binned = sc.bins(data=events,
dim=_event_dimension,
begin=begins,
end=ends)
except sc.SliceError:
raise BadSource(
f"Event index in NXEvent at {group.name}/event_index was not"
f" ordered. The index must be ordered to load pulse times.")
if not quiet:
print(f"Loaded {len(event_time_offset)} events from "
f"{nexus.get_name(group)} containing {num_event} events")
return sc.DataArray(data=binned,
coords={'event_time_zero': event_time_zero})
def test_variable_binned_dataset():
d = sc.Dataset({'a': array_1d, 'b': array_1d})
binned = sc.bins(dim='x', data=d)
check_roundtrip(binned)
def test_variable_binned_data_array():
binned = sc.bins(dim='x', data=array_1d)
check_roundtrip(binned)
def test_variable_binned_variable():
begin = sc.Variable(dims=['y'], values=[0, 3], dtype=sc.dtype.int64)
end = sc.Variable(dims=['y'], values=[3, 4], dtype=sc.dtype.int64)
binned = sc.bins(begin=begin, end=end, dim='x', data=x)
check_roundtrip(binned)
def convert_EventWorkspace_to_data_array(ws,
load_pulse_times=True,
advanced_geometry=False,
load_run_logs=True,
**ignored):
dim, unit = validate_and_get_unit(ws.getAxis(0).getUnit())
spec_dim, spec_coord = init_spec_axis(ws)
nHist = ws.getNumberHistograms()
_, data_unit = validate_and_get_unit(ws.YUnit(), allow_empty=True)
n_event = ws.getNumberEvents()
coord = sc.zeros(dims=['event'],
shape=[n_event],
unit=unit,
dtype=sc.DType.float64)
weights = sc.ones(dims=['event'],
shape=[n_event],
unit=data_unit,
dtype=sc.DType.float32,
with_variances=True)
pulse_times = sc.empty(dims=['event'],
shape=[n_event],
dtype=sc.DType.datetime64,
unit=sc.units.ns) if load_pulse_times else None
begins = sc.zeros(dims=[spec_dim, dim],
shape=[nHist, 1],
dtype=sc.DType.int64)
ends = begins.copy()
current = 0
for i in range(nHist):
sp = ws.getSpectrum(i)
size = sp.getNumberEvents()
coord['event', current:current + size].values = sp.getTofs()
if load_pulse_times:
pulse_times['event', current:current +
size].values = sp.getPulseTimesAsNumpy()
if _contains_weighted_events(sp):
weights['event', current:current + size].values = sp.getWeights()
weights['event',
current:current + size].variances = sp.getWeightErrors()
begins.values[i] = current
ends.values[i] = current + size
current += size
proto_events = {'data': weights, 'coords': {dim: coord}}
if load_pulse_times:
proto_events["coords"]["pulse_time"] = pulse_times
events = sc.DataArray(**proto_events)
coords_labs_data = _convert_MatrixWorkspace_info(
ws, advanced_geometry=advanced_geometry, load_run_logs=load_run_logs)
# For now we ignore potential finer bin edges to avoid creating too many
# bins. Use just a single bin along dim and use extents given by workspace
# edges.
# TODO If there are events outside edges this might create bins with
# events that are not within bin bounds. Consider using `bin` instead
# of `bins`?
edges = coords_labs_data['coords'][dim]
# Using range slice of thickness 1 to avoid transposing 2-D coords
coords_labs_data['coords'][dim] = sc.concat(
[edges[dim, :1], edges[dim, -1:]], dim)
coords_labs_data["data"] = sc.bins(begin=begins,
end=ends,
dim='event',
data=events)
return sc.DataArray(**coords_labs_data)
def test_bins_fail_only_end():
data = sc.Variable(dims=['x'], values=[1, 2, 3, 4])
end = sc.Variable(dims=['y'], values=[1, 3], dtype=sc.dtype.int64)
with pytest.raises(RuntimeError):
sc.bins(end=end, dim='x', data=data)
