def _create_empty_events_data_array( tof_dtype: Any = np.int64, tof_unit: Union[str, sc.Unit] = "ns", detector_id_dtype: Any = np.int32) -> sc.DataArray: data = sc.DataArray(data=sc.empty(dims=[_event_dimension], shape=[0], unit='counts', with_variances=True, dtype=np.float32), coords={ _time_of_flight: sc.empty(dims=[_event_dimension], shape=[0], dtype=tof_dtype, unit=tof_unit), _detector_dimension: sc.empty(dims=[_event_dimension], shape=[0], dtype=detector_id_dtype), }) indices = sc.array(dims=[_pulse_dimension], values=[], dtype='int64') return sc.DataArray(data=sc.bins(begin=indices, end=indices, dim=_event_dimension, data=data), coords={ 'pulse_time': sc.zeros(dims=[_pulse_dimension], shape=[0], dtype='datetime64', unit='ns') })
def test_bins_default_begin_end(): data = sc.Variable(dims=['x'], values=[1, 2, 3, 4]) var = sc.bins(dim='x', data=data) assert var.dims == data.dims assert var.shape == data.shape for i in range(4): assert sc.is_equal(var['x', i].value, data['x', i])
def test_bins_default_end(): data = sc.Variable(dims=['x'], values=[1, 2, 3, 4]) begin = sc.Variable(dims=['y'], values=[1, 3], dtype=sc.dtype.int64) var = sc.bins(begin=begin, dim='x', data=data) assert var.dims == begin.dims assert var.shape == begin.shape assert sc.is_equal(var['y', 0].value, data['x', 1]) assert sc.is_equal(var['y', 1].value, data['x', 3])
def test_bins(): data = sc.Variable(dims=['x'], values=[1, 2, 3, 4]) begin = sc.Variable(dims=['y'], values=[0, 2], dtype=sc.dtype.int64) end = sc.Variable(dims=['y'], values=[2, 4], dtype=sc.dtype.int64) var = sc.bins(begin=begin, end=end, dim='x', data=data) assert var.dims == begin.dims assert var.shape == begin.shape assert sc.is_equal(var['y', 0].value, data['x', 0:2]) assert sc.is_equal(var['y', 1].value, data['x', 2:4])
def make_tof_binned_events(): buffer = sc.DataArray(sc.zeros(dims=['event'], shape=[7], dtype=float), coords={ 'tof': sc.array(dims=['event'], values=[ 1000.0, 3000.0, 2000.0, 4000.0, 5000.0, 6000.0, 3000.0 ], unit='us') }) return sc.bins(data=buffer, dim='event', begin=sc.array(dims=['spectrum'], values=[0, 4], dtype='int64'), end=sc.array(dims=['spectrum'], values=[4, 7], dtype='int64'))
def test_load_events_bins(): with in_memory_nexus_file_with_event_data() as nexus_file: event_data_groups = find_by_nx_class("NXevent_data", nexus_file) # Load only the first event data group we found event_group = event_data_groups[0] event_index_np = event_group["event_index"][...] event_time_offset = sc.Variable( ['event'], values=event_group["event_time_offset"][...]) event_id = sc.Variable(['event'], values=event_group["event_id"][...]) event_index = sc.Variable(['pulse'], values=event_index_np, dtype=np.int64) event_time_zero = sc.Variable( ['pulse'], values=event_group["event_time_zero"][...]) # Calculate the end index for each pulse # The end index for a pulse is the start index of the next pulse end_indices = event_index_np.astype(np.int64) end_indices = np.roll(end_indices, -1) end_indices[-1] = event_id.shape[0] end_indices = sc.Variable(['pulse'], values=end_indices, dtype=np.int64) # Weights are not stored in NeXus, so use 1s weights = sc.Variable(['event'], values=np.ones(event_id.shape), dtype=np.float32) data = sc.DataArray(data=weights, coords={ 'tof': event_time_offset, 'detector-id': event_id }) events = sc.DataArray(data=sc.bins(begin=event_index, end=end_indices, dim='event', data=data), coords={'pulse-time': event_time_zero}) assert events.dims == event_index.dims assert events.shape == event_index.shape assert sc.is_equal(events['pulse', 0].value.coords['detector-id'], data.coords['detector-id']['event', 0:3])
def test_convert_binned_events_converted(target): tof = make_test_data(coords=('Ltotal', 'two_theta'), dataset=True) del tof['counts'] tof['events'] = make_tof_binned_events() # Construct events with all coords. binned_tof = tof['events'].copy() for name in ('Ltotal', 'two_theta'): binned_tof.bins.coords[name] = sc.bins_like(binned_tof, tof.coords[name]) dense_tof = binned_tof.bins.constituents['data'] expected = scn.convert(dense_tof, origin='tof', target=target, scatter=True) for intermediate in ('Ltotal', 'two_theta'): expected.attrs.pop(intermediate, None) expected.coords.pop(intermediate, None) expected = sc.bins(**{**binned_tof.bins.constituents, 'data': expected}) res = scn.convert(tof, origin='tof', target=target, scatter=True) assert sc.identical(res['events'].data, expected)
def _create_empty_event_data(detectors: List[DetectorData]): """ If any NXdetector groups had pixel position data but no events then add an empty data array to make it easier to concatenate the data from different groups """ empty_events = None detector_id_dtype = None for data in detectors: if data.event_data is not None: # If any event data were loaded then use an empty data # array with the same data types constituents = data.event_data.bins.constituents constituents['begin'] = sc.zeros_like(constituents['begin']) constituents['end'] = sc.zeros_like(constituents['end']) constituents['data'] = constituents['data'][_event_dimension, 0:0].copy() empty_events = data.event_data.copy(deep=False) empty_events.data = sc.bins(**constituents) break elif data.detector_ids is not None: detector_id_dtype = data.detector_ids.dtype if empty_events is None: if detector_id_dtype is None: # Create empty data array with types/unit matching streamed event # data, this avoids need to convert to concatenate with event data # arriving from stream empty_events = _create_empty_events_data_array( np.int64, "ns", np.int32) else: # If detector_ids were loaded then match the type used for those empty_events = _create_empty_events_data_array( np.int64, "ns", detector_id_dtype) for data in detectors: if data.event_data is None: data.event_data = empty_events
def _load_event_group(group: Group, nexus: LoadFromNexus, quiet: bool, select=tuple()) -> DetectorData: _check_for_missing_fields(group, nexus) index = to_plain_index([_pulse_dimension], select) def shape(name): return nexus.get_shape(nexus.get_dataset_from_group(group, name)) max_index = shape("event_index")[0] single = False if index is Ellipsis or index == tuple(): last_loaded = False else: if isinstance(index, int): single = True start, stop, _ = slice(index, None).indices(max_index) if start == stop: raise IndexError('Index {start} is out of range') index = slice(start, start + 1) start, stop, stride = index.indices(max_index) if stop + stride > max_index: last_loaded = False else: stop += stride last_loaded = True index = slice(start, stop, stride) event_index = nexus.load_dataset_from_group_as_numpy_array( group, "event_index", index) event_time_zero = _load_event_time_zero(group, nexus, index) num_event = shape("event_time_offset")[0] # Some files contain uint64 "max" indices, which turn into negatives during # conversion to int64. This is a hack to get arround this. event_index[event_index < 0] = num_event if len(event_index) > 0: event_select = slice(event_index[0], event_index[-1] if last_loaded else num_event) else: event_select = slice(None) if nexus.dataset_in_group(group, "event_id")[0]: event_id = nexus.load_dataset(group, "event_id", [_event_dimension], index=event_select) else: event_id = None event_time_offset = nexus.load_dataset(group, "event_time_offset", [_event_dimension], index=event_select) # Weights are not stored in NeXus, so use 1s weights = sc.ones(dims=[_event_dimension], shape=event_time_offset.shape, unit='counts', dtype=np.float32, with_variances=True) events = sc.DataArray(data=weights, coords={'event_time_offset': event_time_offset}) if event_id is not None: events.coords['event_id'] = event_id if not last_loaded: event_index = np.append(event_index, num_event) else: # Not a bin-edge coord, all events in bin are associated with same (previous) # pulse time value event_time_zero = event_time_zero[:-1] event_index = sc.array(dims=[_pulse_dimension], values=event_index, dtype=sc.DType.int64) event_index -= event_index.min() # There is some variation in the last recorded event_index in files from different # institutions. We try to make sure here that it is what would be the first index of # the next pulse. In other words, ensure that event_index includes the bin edge for # the last pulse. if single: begins = event_index[_pulse_dimension, 0] ends = event_index[_pulse_dimension, 1] event_time_zero = event_time_zero[_pulse_dimension, 0] else: begins = event_index[_pulse_dimension, :-1] ends = event_index[_pulse_dimension, 1:] try: binned = sc.bins(data=events, dim=_event_dimension, begin=begins, end=ends) except sc.SliceError: raise BadSource( f"Event index in NXEvent at {group.name}/event_index was not" f" ordered. The index must be ordered to load pulse times.") if not quiet: print(f"Loaded {len(event_time_offset)} events from " f"{nexus.get_name(group)} containing {num_event} events") return sc.DataArray(data=binned, coords={'event_time_zero': event_time_zero})
def test_variable_binned_dataset(): d = sc.Dataset({'a': array_1d, 'b': array_1d}) binned = sc.bins(dim='x', data=d) check_roundtrip(binned)
def test_variable_binned_data_array(): binned = sc.bins(dim='x', data=array_1d) check_roundtrip(binned)
def test_variable_binned_variable(): begin = sc.Variable(dims=['y'], values=[0, 3], dtype=sc.dtype.int64) end = sc.Variable(dims=['y'], values=[3, 4], dtype=sc.dtype.int64) binned = sc.bins(begin=begin, end=end, dim='x', data=x) check_roundtrip(binned)
def convert_EventWorkspace_to_data_array(ws, load_pulse_times=True, advanced_geometry=False, load_run_logs=True, **ignored): dim, unit = validate_and_get_unit(ws.getAxis(0).getUnit()) spec_dim, spec_coord = init_spec_axis(ws) nHist = ws.getNumberHistograms() _, data_unit = validate_and_get_unit(ws.YUnit(), allow_empty=True) n_event = ws.getNumberEvents() coord = sc.zeros(dims=['event'], shape=[n_event], unit=unit, dtype=sc.DType.float64) weights = sc.ones(dims=['event'], shape=[n_event], unit=data_unit, dtype=sc.DType.float32, with_variances=True) pulse_times = sc.empty(dims=['event'], shape=[n_event], dtype=sc.DType.datetime64, unit=sc.units.ns) if load_pulse_times else None begins = sc.zeros(dims=[spec_dim, dim], shape=[nHist, 1], dtype=sc.DType.int64) ends = begins.copy() current = 0 for i in range(nHist): sp = ws.getSpectrum(i) size = sp.getNumberEvents() coord['event', current:current + size].values = sp.getTofs() if load_pulse_times: pulse_times['event', current:current + size].values = sp.getPulseTimesAsNumpy() if _contains_weighted_events(sp): weights['event', current:current + size].values = sp.getWeights() weights['event', current:current + size].variances = sp.getWeightErrors() begins.values[i] = current ends.values[i] = current + size current += size proto_events = {'data': weights, 'coords': {dim: coord}} if load_pulse_times: proto_events["coords"]["pulse_time"] = pulse_times events = sc.DataArray(**proto_events) coords_labs_data = _convert_MatrixWorkspace_info( ws, advanced_geometry=advanced_geometry, load_run_logs=load_run_logs) # For now we ignore potential finer bin edges to avoid creating too many # bins. Use just a single bin along dim and use extents given by workspace # edges. # TODO If there are events outside edges this might create bins with # events that are not within bin bounds. Consider using `bin` instead # of `bins`? edges = coords_labs_data['coords'][dim] # Using range slice of thickness 1 to avoid transposing 2-D coords coords_labs_data['coords'][dim] = sc.concat( [edges[dim, :1], edges[dim, -1:]], dim) coords_labs_data["data"] = sc.bins(begin=begins, end=ends, dim='event', data=events) return sc.DataArray(**coords_labs_data)
def test_bins_fail_only_end(): data = sc.Variable(dims=['x'], values=[1, 2, 3, 4]) end = sc.Variable(dims=['y'], values=[1, 3], dtype=sc.dtype.int64) with pytest.raises(RuntimeError): sc.bins(end=end, dim='x', data=data)