def align_zlp_xdata(src_xdata: DataAndMetadata.DataAndMetadata, progress_fn=None, method='com', roi: typing.Optional[API_1_0.Graphic]=None, ref_index: int=0) -> typing.Tuple[typing.Optional[DataAndMetadata.DataAndMetadata], typing.Optional[DataAndMetadata.DataAndMetadata]]:
# check to make sure it is suitable for this algorithm
# if (src_xdata.is_datum_1d and (src_xdata.is_sequence or src_xdata.is_collection)) or (src_xdata.is_datum_2d and not (src_xdata.is_sequence or src_xdata.is_collection)):
if src_xdata.is_datum_1d or (src_xdata.is_datum_2d and not (src_xdata.is_sequence or src_xdata.is_collection)):
# get the numpy array and create the destination data
src_data = src_xdata.data
assert src_data is not None
d_rank = 1
src_shape = tuple(src_xdata.data_shape)
d_shape = src_shape[-d_rank:]
if roi and roi.graphic_type == "interval-graphic":
data_slice = slice(int(roi.start * d_shape[0]), int(roi.end * d_shape[0]))
elif roi and roi.graphic_type == "rect-graphic":
ref_index = int(roi.bounds[0][0] * src_shape[0])
data_slice = slice(int(roi.bounds[0][1] * d_shape[0]), int((roi.bounds[0][1] + roi.bounds[1][1]) * d_shape[0]))
else:
data_slice = slice(0, None)
flat_src_data = numpy.reshape(src_data, (-1,) + d_shape)
flat_dst_data = numpy.zeros_like(flat_src_data)
flat_pos_data = numpy.zeros(flat_src_data.shape[0], dtype=numpy.float32)
if method == "com":
get_position_fn = ZLP_Analysis.estimate_zlp_amplitude_position_width_com
interpolation_order = 1
elif method == "fit":
get_position_fn = ZLP_Analysis.estimate_zlp_amplitude_position_width_fit_spline
interpolation_order = 1
elif method == "max":
get_position_fn = lambda data: (None, numpy.argmax(data), None)
interpolation_order = 0
else:
raise ValueError(f"Method {method} is not supported. Allowed options are 'com', 'fit' and 'max'.")
# use this as the reference position. all other spectra will be aligned to this one.
ref_pos = get_position_fn(flat_src_data[ref_index, data_slice])[1]
# put the first spectrum in the result
flat_dst_data[ref_index] = flat_src_data[ref_index]
# loop over all non-datum dimensions linearly
for i in range(len(flat_src_data)):
if i == ref_index:
continue
# the algorithm in this early version is to find the max value
mx_pos = get_position_fn(flat_src_data[i, data_slice])[1]
# fallback to simple max if get_position_fun failed
if mx_pos is numpy.nan:
mx_pos = numpy.argmax(flat_src_data[i, data_slice])
# determine the offset and apply it
offset = ref_pos - mx_pos
flat_dst_data[i] = scipy.ndimage.shift(flat_src_data[i], offset, order=interpolation_order)
flat_pos_data[i] = -offset
# every row, report progress (will also work for a sequence or 1d collection
# because there we have only 1 row anyways)
if i % src_shape[1] == 0 and callable(progress_fn):
progress_fn(i//src_shape[1])
dimensional_calibrations = copy.deepcopy(src_xdata.dimensional_calibrations)
energy_calibration = dimensional_calibrations[-1]
energy_calibration.offset = -(ref_pos + 0.5) * energy_calibration.scale
dimensional_calibrations = list(dimensional_calibrations[:-1]) + [energy_calibration]
shift_calibration = copy.copy(energy_calibration)
shift_calibration.offset = 0
# dst_data is complete. construct xdata with correct calibration and data descriptor.
data_descriptor = DataAndMetadata.DataDescriptor(src_xdata.is_sequence, src_xdata.collection_dimension_count, src_xdata.datum_dimension_count)
shift_xdata = None
if flat_pos_data.size > 1:
shift_xdata = DataAndMetadata.new_data_and_metadata(flat_pos_data.reshape(src_shape[:-d_rank]), shift_calibration, dimensional_calibrations[:-d_rank])
return (DataAndMetadata.new_data_and_metadata(flat_dst_data.reshape(src_shape), src_xdata.intensity_calibration, dimensional_calibrations, data_descriptor=data_descriptor),
shift_xdata)
return None, None
def get_frame_data(self, readout_area: Geometry.IntRect, binning_shape: Geometry.IntSize, exposure_s: float, scan_context, parked_probe_position) -> DataAndMetadata.DataAndMetadata:
# check if one of the arguments has changed since last call
new_frame_settings = [readout_area, binning_shape, exposure_s, copy.deepcopy(scan_context)]
if new_frame_settings != self._last_frame_settings:
self._needs_recalculation = True
if self.instrument.sample != self.__last_sample:
self._needs_recalculation = True
self._last_frame_settings = new_frame_settings
if self._needs_recalculation or self.__cached_frame is None:
#print("recalculating frame")
height = readout_area.height
width = readout_area.width
offset_m = self.instrument.stage_position_m
full_fov_nm = self.__stage_size_nm
fov_size_nm = Geometry.FloatSize(full_fov_nm * height / self._sensor_dimensions.height, full_fov_nm * width / self._sensor_dimensions.width)
center_nm = Geometry.FloatPoint(full_fov_nm * (readout_area.center.y / self._sensor_dimensions.height- 0.5), full_fov_nm * (readout_area.center.x / self._sensor_dimensions.width - 0.5))
size = Geometry.IntSize(height, width)
data = numpy.zeros((height, width), numpy.float32)
# features will be positive values; thickness can be simulated by subtracting the features from the
# vacuum value. the higher the vacuum value, the thinner (i.e. less contribution from features).
thickness_param = 100
if not self.instrument.is_blanked:
self.instrument.sample.plot_features(data, offset_m, fov_size_nm, Geometry.FloatPoint(), center_nm, size)
data = thickness_param - data
data = self._get_binned_data(data, binning_shape)
self.__last_sample = self.instrument.sample
if not self.instrument.is_blanked:
probe_position = Geometry.FloatPoint(0.5, 0.5)
if self.instrument.probe_state == "scanning":
probe_position = self.instrument.live_probe_position
elif self.instrument.probe_state == "parked" and parked_probe_position is not None:
probe_position = parked_probe_position
scan_offset = Geometry.FloatPoint()
if scan_context.is_valid and probe_position is not None:
scan_offset = Geometry.FloatPoint(
y=probe_position[0] * scan_context.fov_size_nm[0] - scan_context.fov_size_nm[0] / 2,
x=probe_position[1] * scan_context.fov_size_nm[1] - scan_context.fov_size_nm[1] / 2)
scan_offset = scan_offset*1e-9
theta = self.__tv_pixel_angle * self._sensor_dimensions.height / 2 # half angle on camera
aberrations = dict()
aberrations["height"] = data.shape[0]
aberrations["width"] = data.shape[1]
aberrations["theta"] = theta
aberrations["c0a"] = self.instrument.GetVal2D("beam_shift_m").x + scan_offset[1]
aberrations["c0b"] = self.instrument.GetVal2D("beam_shift_m").y + scan_offset[0]
aberrations["c10"] = self.instrument.GetVal("C10Control")
aberrations["c12a"] = self.instrument.GetVal2D("C12Control").x
aberrations["c12b"] = self.instrument.GetVal2D("C12Control").y
aberrations["c21a"] = self.instrument.GetVal2D("C21Control").x
aberrations["c21b"] = self.instrument.GetVal2D("C21Control").y
aberrations["c23a"] = self.instrument.GetVal2D("C23Control").x
aberrations["c23b"] = self.instrument.GetVal2D("C23Control").y
aberrations["c30"] = self.instrument.GetVal("C30Control")
aberrations["c32a"] = self.instrument.GetVal2D("C32Control").x
aberrations["c32b"] = self.instrument.GetVal2D("C32Control").y
aberrations["c34a"] = self.instrument.GetVal2D("C34Control").x
aberrations["c34b"] = self.instrument.GetVal2D("C34Control").y
data = self.__aberrations_controller.apply(aberrations, data)
if self.instrument.GetVal("S_VOA") > 0:
self._draw_aperture(data, binning_shape)
elif self.instrument.GetVal("S_MOA") > 0:
self._draw_aperture(data, binning_shape, enlarge_by=0.1)
intensity_calibration = Calibration.Calibration(units="counts")
dimensional_calibrations = self.get_dimensional_calibrations(readout_area, binning_shape)
self.__cached_frame = DataAndMetadata.new_data_and_metadata(data.astype(numpy.float32), intensity_calibration=intensity_calibration, dimensional_calibrations=dimensional_calibrations)
self.__data_scale = self.get_total_counts(exposure_s) / (data.shape[0] * data.shape[1] * thickness_param)
self._needs_recalculation = False
self.noise.poisson_level = self.__data_scale
return self.noise.apply(self.__cached_frame * self.__data_scale)
def acquire_series(
number_frames: int,
energy_offset: float,
dark_ref_enabled: bool,
dark_ref_data: typing.Optional[_NDArray],
task_object: typing.Optional[Task.TaskContextManager] = None
) -> DataItem.DataItem:
logging.info("Starting image acquisition.")
# grab one frame to get image size
first_xdatas = camera.get_next_xdatas_to_start()
first_xdata = first_xdatas[0] if first_xdatas else None
first_data = first_xdata.data if first_xdata else None
assert first_xdata is not None
assert first_data is not None
# Initialize an empty stack to fill with acquired data
image_stack_data: numpy.typing.NDArray[numpy.float_] = numpy.empty(
(number_frames, first_data.shape[0], first_data.shape[1]),
dtype=float)
reference_energy = stem_controller.GetVal(energy_adjust_control)
# loop through the frames and fill in the empty stack from the
# camera
for frame_index in range(number_frames):
if energy_offset == 0.:
set_offset_energy(energy_offset, 1)
# use next frame to start to make sure we're getting a
# frame with the new energy offset
if frame_index == 0:
xdatas = camera.get_next_xdatas_to_start()
xdata0 = xdatas[0] if xdatas else None
data = xdata0.data if xdata0 else None
assert data is not None
image_stack_data[frame_index] = data
else:
# grab the following frames
xdatas = camera.get_next_xdatas_to_finish()
xdata0 = xdatas[0] if xdatas else None
data = xdata0.data if xdata0 else None
assert data is not None
image_stack_data[frame_index] = data
if task_object is not None:
# Update the task panel with the progress
task_object.update_progress(
_("Grabbing EELS data frame {}.").format(frame_index +
1),
(frame_index + 1, number_frames), None)
# Blank the beam
stem_controller.SetValWait(blank_control, 1.0, 200)
# load dark ref file
if dark_ref_enabled:
# User desires a dark reference to be applied
dark_sum: typing.Optional[_NDArray] = None
if dark_ref_data is not None:
# User has provided a valid dark reference file
if task_object is not None:
task_object.update_progress(
_("Applying dark reference"), None)
dark_sum = dark_ref_data
else:
# User has not provided a valid dark reference, so a
# dark reference will be acquired
dark_sum = acquire_dark(number_frames, sleep_time,
task_object)
# Apply dark reference data to the image stack
if dark_sum is not None:
image_stack_data -= dark_sum / number_frames
stem_controller.SetVal(energy_adjust_control, reference_energy)
dimension_calibration0 = first_xdata.dimensional_calibrations[0]
dimension_calibration1 = first_xdata.dimensional_calibrations[1]
# TODO: replace frame calibration with acquisition time
# (this is effectively chronospectroscopy before the sum)
sequence_calibration = Calibration.Calibration(units="frame")
# numpy zero array is dummy data
image_stack_data_item = DataItem.DataItem(numpy.zeros((8, 8)))
# Insert acquired data into a calibrated image stack
image_stack_data_item.set_xdata(
DataAndMetadata.new_data_and_metadata(
image_stack_data,
dimensional_calibrations=[
sequence_calibration, dimension_calibration0,
dimension_calibration1
],
data_descriptor=DataAndMetadata.DataDescriptor(True, 0,
2)))
return image_stack_data_item
def execute(self, eels_spectrum_data_item, background_model,
fit_interval_graphics, **kwargs) -> None:
try:
spectrum_xdata = eels_spectrum_data_item.xdata
assert spectrum_xdata.is_datum_1d
assert spectrum_xdata.datum_dimensional_calibrations[
0].units == "eV"
eels_spectrum_xdata = spectrum_xdata
# fit_interval_graphics.interval returns normalized coordinates. create calibrated intervals.
fit_intervals = list()
for fit_interval_graphic in fit_interval_graphics:
fit_interval = Calibration.CalibratedInterval(
Calibration.Coordinate(
Calibration.CoordinateType.NORMALIZED,
fit_interval_graphic.interval[0]),
Calibration.Coordinate(
Calibration.CoordinateType.NORMALIZED,
fit_interval_graphic.interval[1]))
fit_intervals.append(fit_interval)
fit_minimum = min(
[fit_interval.start.value for fit_interval in fit_intervals])
signal_interval = Calibration.CalibratedInterval(
Calibration.Coordinate(Calibration.CoordinateType.NORMALIZED,
fit_minimum),
Calibration.Coordinate(Calibration.CoordinateType.NORMALIZED,
1.0))
reference_frame = Calibration.ReferenceFrameAxis(
eels_spectrum_xdata.datum_dimensional_calibrations[0],
eels_spectrum_xdata.datum_dimension_shape[0])
signal_xdata = Core.get_calibrated_interval_slice(
eels_spectrum_xdata, reference_frame, signal_interval)
background_xdata = None
subtracted_xdata = None
if background_model._data_structure.entity:
entity_id = background_model._data_structure.entity.entity_type.entity_id
for component in Registry.get_components_by_type(
"background-model"):
if entity_id == component.background_model_id:
fit_result = component.fit_background(
spectrum_xdata=spectrum_xdata,
fit_intervals=fit_intervals,
background_interval=signal_interval)
background_xdata = fit_result["background_model"]
# use 'or' to avoid doing subtraction if subtracted_spectrum already present
subtracted_xdata = fit_result.get(
"subtracted_spectrum",
None) or Core.calibrated_subtract_spectrum(
spectrum_xdata, background_xdata)
if background_xdata is None:
background_xdata = DataAndMetadata.new_data_and_metadata(
numpy.zeros_like(signal_xdata.data),
intensity_calibration=signal_xdata.intensity_calibration,
dimensional_calibrations=signal_xdata.
dimensional_calibrations)
if subtracted_xdata is None:
subtracted_xdata = DataAndMetadata.new_data_and_metadata(
signal_xdata.data,
intensity_calibration=signal_xdata.intensity_calibration,
dimensional_calibrations=signal_xdata.
dimensional_calibrations)
self.__background_xdata = background_xdata
self.__subtracted_xdata = subtracted_xdata
except Exception as e:
import traceback
print(traceback.format_exc())
print(e)
raise
def load_image(file) -> DataAndMetadata.DataAndMetadata:
"""
Loads the image from the file-like object or string file.
If file is a string, the file is opened and then read.
Returns a numpy ndarray of our best guess for the most important image
in the file.
"""
if isinstance(file, str) or isinstance(file, str):
with open(file, "rb") as f:
return load_image(f)
dmtag = parse_dm3.parse_dm_header(file)
dmtag = fix_strings(dmtag)
# display_keys(dmtag)
img_index = -1
image_tags = dmtag['ImageList'][img_index]
data = imagedatadict_to_ndarray(image_tags['ImageData'])
calibrations = []
calibration_tags = image_tags['ImageData'].get('Calibrations', dict())
for dimension in calibration_tags.get('Dimension', list()):
origin, scale, units = dimension.get('Origin', 0.0), dimension.get(
'Scale', 1.0), dimension.get('Units', str())
calibrations.append((-origin * scale, scale, units))
calibrations = tuple(reversed(calibrations))
if len(data.shape) == 3 and data.dtype != numpy.uint8:
if image_tags['ImageTags'].get('Meta Data', dict()).get(
"Format", str()).lower() in ("spectrum", "spectrum image"):
if data.shape[1] == 1:
data = numpy.squeeze(data, 1)
data = numpy.moveaxis(data, 0, 1)
data_descriptor = DataAndMetadata.DataDescriptor(False, 1, 1)
calibrations = (calibrations[2], calibrations[0])
else:
data = numpy.moveaxis(data, 0, 2)
data_descriptor = DataAndMetadata.DataDescriptor(False, 2, 1)
calibrations = tuple(calibrations[1:]) + (calibrations[0], )
else:
data_descriptor = DataAndMetadata.DataDescriptor(False, 1, 2)
elif len(data.shape) == 4 and data.dtype != numpy.uint8:
# data = numpy.moveaxis(data, 0, 2)
data_descriptor = DataAndMetadata.DataDescriptor(False, 2, 2)
elif data.dtype == numpy.uint8:
data_descriptor = DataAndMetadata.DataDescriptor(
False, 0, len(data.shape[:-1]))
else:
data_descriptor = DataAndMetadata.DataDescriptor(
False, 0, len(data.shape))
brightness = calibration_tags.get('Brightness', dict())
origin, scale, units = brightness.get('Origin', 0.0), brightness.get(
'Scale', 1.0), brightness.get('Units', str())
intensity = -origin * scale, scale, units
timestamp = None
timezone = None
timezone_offset = None
title = image_tags.get('Name')
properties = dict()
if 'ImageTags' in image_tags:
voltage = image_tags['ImageTags'].get('ImageScanned',
dict()).get('EHT', dict())
if voltage:
properties.setdefault("hardware_source", dict())["autostem"] = {
"high_tension_v": float(voltage)
}
dm_metadata_signal = image_tags['ImageTags'].get('Meta Data',
dict()).get('Signal')
if dm_metadata_signal and dm_metadata_signal.lower() == "eels":
properties.setdefault("hardware_source",
dict())["signal_type"] = dm_metadata_signal
if image_tags['ImageTags'].get('Meta Data', dict()).get(
"Format", str()).lower() in ("spectrum", "spectrum image"):
data_descriptor.collection_dimension_count += data_descriptor.datum_dimension_count - 1
data_descriptor.datum_dimension_count = 1
if image_tags['ImageTags'].get('Meta Data', dict()).get(
"IsSequence",
False) and data_descriptor.collection_dimension_count > 0:
data_descriptor.is_sequence = True
data_descriptor.collection_dimension_count -= 1
timestamp_str = image_tags['ImageTags'].get("Timestamp")
if timestamp_str:
timestamp = get_datetime_from_timestamp_str(timestamp_str)
timezone = image_tags['ImageTags'].get("Timezone")
timezone_offset = image_tags['ImageTags'].get("TimezoneOffset")
# to avoid having duplicate copies in Swift, get rid of these tags
image_tags['ImageTags'].pop("Timestamp", None)
image_tags['ImageTags'].pop("Timezone", None)
image_tags['ImageTags'].pop("TimezoneOffset", None)
# put the image tags into properties
properties.update(image_tags['ImageTags'])
dimensional_calibrations = [
Calibration.Calibration(c[0], c[1], c[2]) for c in calibrations
]
while len(dimensional_calibrations
) < data_descriptor.expected_dimension_count:
dimensional_calibrations.append(Calibration.Calibration())
intensity_calibration = Calibration.Calibration(intensity[0], intensity[1],
intensity[2])
return DataAndMetadata.new_data_and_metadata(
data,
data_descriptor=data_descriptor,
dimensional_calibrations=dimensional_calibrations,
intensity_calibration=intensity_calibration,
metadata=properties,
timestamp=timestamp,
timezone=timezone,
timezone_offset=timezone_offset)
def __fit_background(
self, spectrum_xdata: DataAndMetadata.DataAndMetadata,
fit_intervals: typing.Sequence[Calibration.CalibratedInterval],
background_interval: Calibration.CalibratedInterval
) -> DataAndMetadata.DataAndMetadata:
reference_frame = Calibration.ReferenceFrameAxis(
spectrum_xdata.datum_dimensional_calibrations[0],
spectrum_xdata.datum_dimension_shape[0])
# fit polynomial to the data
xs = numpy.concatenate([
Core.get_calibrated_interval_domain(reference_frame, fit_interval)
for fit_interval in fit_intervals
])
if len(fit_intervals) > 1:
ys = numpy.concatenate([
Core.get_calibrated_interval_slice(spectrum_xdata,
reference_frame,
fit_interval).data
for fit_interval in fit_intervals
])
else:
ys = Core.get_calibrated_interval_slice(spectrum_xdata,
reference_frame,
fit_intervals[0]).data
# generate background model data from the series
n = reference_frame.convert_to_pixel(
background_interval.end
).int_value - reference_frame.convert_to_pixel(
background_interval.start).int_value
interval_start = reference_frame.convert_to_calibrated(
background_interval.start).value
interval_end = reference_frame.convert_to_calibrated(
background_interval.end).value
interval_end -= (interval_end - interval_start
) / n # n samples at the left edges of each pixel
calibration = copy.deepcopy(
spectrum_xdata.datum_dimensional_calibrations[0])
calibration.offset = reference_frame.convert_to_calibrated(
background_interval.start).value
fs = numpy.linspace(interval_start, interval_end, n)
if spectrum_xdata.is_navigable:
calibrations = list(
copy.deepcopy(
spectrum_xdata.navigation_dimensional_calibrations)) + [
calibration
]
yss = numpy.reshape(ys, (numpy.product(ys.shape[:-1]), ) +
(ys.shape[-1], ))
fit_data = self._perform_fits(xs, yss, fs)
data_descriptor = DataAndMetadata.DataDescriptor(
False, spectrum_xdata.navigation_dimension_count,
spectrum_xdata.datum_dimension_count)
background_xdata = DataAndMetadata.new_data_and_metadata(
numpy.reshape(fit_data, ys.shape[:-1] + (n, )),
data_descriptor=data_descriptor,
dimensional_calibrations=calibrations,
intensity_calibration=spectrum_xdata.intensity_calibration)
else:
poly_data = self._perform_fit(xs, ys, fs)
background_xdata = DataAndMetadata.new_data_and_metadata(
poly_data,
dimensional_calibrations=[calibration],
intensity_calibration=spectrum_xdata.intensity_calibration)
return background_xdata
async def grab(self, document_controller, hardware_source, do_acquire):
# this is an async method meaning that it will execute until it calls await, at which time
# it will let other parts of the software run until the awaited function finishes. in this
# case, waiting for acquired data and grabbing the last frames are run in a thread.
assert document_controller
assert hardware_source
event_loop = document_controller.event_loop
self.cancel_event.clear()
self.state.value = "running"
self.progress_model.value = 0
frame_count = self.frame_count_model.value
was_playing = hardware_source.is_playing
success_ref = [True]
xdata_group_list = list()
def exec_acquire():
# this will execute in a thread; the enclosing async routine will continue when it finishes
try:
start_time = time.time()
max_wait_time = max(hardware_source.get_current_frame_time() * 1.5, 3)
while not hardware_source.is_playing:
if time.time() - start_time > max_wait_time:
success_ref[0] = False
return
time.sleep(0.01)
hardware_source.get_next_xdatas_to_start(max_wait_time * 2) # wait for frame + next frame
for i in range(frame_count):
self.progress_model.value = int(100 * i / frame_count)
if self.cancel_event.is_set():
success_ref[0] = False
break
hardware_source.get_next_xdatas_to_finish(max_wait_time * 2)
except Exception as e:
import traceback
traceback.print_exc()
success_ref[0] = False
if do_acquire:
print("AR: start playing")
hardware_source.start_playing()
print("AR: wait for acquire")
await event_loop.run_in_executor(None, exec_acquire)
print("AR: acquire finished")
def exec_grab():
# this will execute in a thread; the enclosing async routine will continue when it finishes
try:
start_time = time.time()
max_wait_time = max(hardware_source.get_current_frame_time() * 1.5, 3)
while hardware_source.is_playing:
if time.time() - start_time > max_wait_time:
success_ref[0] = False
return
time.sleep(0.01)
data_element_groups = hardware_source.get_buffer_data(-frame_count, frame_count)
for data_element_group in data_element_groups:
if self.cancel_event.is_set():
success_ref[0] = False
break
xdata_group = list()
for data_element in data_element_group:
xdata = ImportExportManager.convert_data_element_to_data_and_metadata(data_element)
xdata_group.append(xdata)
xdata_group_list.append(xdata_group)
self.progress_model.value = 100
except Exception as e:
import traceback
traceback.print_exc()
success_ref[0] = False
if success_ref[0]:
print("AR: stop playing")
hardware_source.stop_playing()
print("AR: grabbing data")
await event_loop.run_in_executor(None, exec_grab)
print("AR: grab finished")
xdata_group = None
if success_ref[0]:
if len(xdata_group_list) > 1:
print("AR: making xdata")
valid_count = 0
examplar_xdata_group = xdata_group_list[-1]
shapes = [xdata.data.shape for xdata in examplar_xdata_group]
dtypes = [xdata.data.dtype for xdata in examplar_xdata_group]
for xdata_group in reversed(xdata_group_list):
shapes_i = [xdata.data.shape for xdata in xdata_group]
dtypes_i = [xdata.data.dtype for xdata in xdata_group]
if shapes_i == shapes and dtypes_i == dtypes:
valid_count += 1
xdata_group = list()
for i, xdata in enumerate(examplar_xdata_group):
intensity_calibration = xdata.intensity_calibration
dimensional_calibrations = [Calibration.Calibration()] + list(xdata.dimensional_calibrations)
data_descriptor = DataAndMetadata.DataDescriptor(True,
xdata.data_descriptor.collection_dimension_count,
xdata.data_descriptor.datum_dimension_count)
data = numpy.vstack(list(xdata_group[i].data for xdata_group in xdata_group_list[-valid_count:])).reshape(valid_count, *shapes[i])
xdata = DataAndMetadata.new_data_and_metadata(data,
intensity_calibration=intensity_calibration,
dimensional_calibrations=dimensional_calibrations,
data_descriptor=data_descriptor,
metadata=xdata.metadata)
xdata_group.append(xdata)
elif len(xdata_group_list) == 1:
xdata_group = xdata_group_list[0]
if xdata_group:
print("AR: making data item")
for xdata in xdata_group:
data_item = DataItem.DataItem(large_format=True)
data_item.ensure_data_source()
data_item.set_xdata(xdata)
channel_name = xdata.metadata.get("hardware_source", dict()).get("channel_name")
channel_ext = (" (" + channel_name + ")") if channel_name else ""
data_item.title = _("Recording of ") + hardware_source.display_name + channel_ext
document_controller.document_model.append_data_item(data_item)
display_item = document_controller.document_model.get_display_item_for_data_item(data_item)
document_controller.show_display_item(display_item)
if was_playing:
print("AR: restarting")
hardware_source.start_playing()
self.state.value = "idle"
self.progress_model.value = 0
print("AR: done")
def convert_data_element_to_data_and_metadata_1(
data_element) -> DataAndMetadata.DataAndMetadata:
"""Convert a data element to xdata. No data copying occurs.
The data element can have the following keys:
data (required)
is_sequence, collection_dimension_count, datum_dimension_count (optional description of the data)
spatial_calibrations (optional list of spatial calibration dicts, scale, offset, units)
intensity_calibration (optional intensity calibration dict, scale, offset, units)
metadata (optional)
properties (get stored into metadata.hardware_source)
one of either timestamp or datetime_modified
if datetime_modified (dst, tz) it is converted and used as timestamp
then timezone gets stored into metadata.description.timezone.
"""
# data. takes ownership.
data = data_element["data"]
dimensional_shape = Image.dimensional_shape_from_data(data)
is_sequence = data_element.get("is_sequence", False)
dimension_count = len(Image.dimensional_shape_from_data(data))
adjusted_dimension_count = dimension_count - (1 if is_sequence else 0)
collection_dimension_count = data_element.get(
"collection_dimension_count",
2 if adjusted_dimension_count in (3, 4) else 0)
datum_dimension_count = data_element.get(
"datum_dimension_count",
adjusted_dimension_count - collection_dimension_count)
data_descriptor = DataAndMetadata.DataDescriptor(
is_sequence, collection_dimension_count, datum_dimension_count)
# dimensional calibrations
dimensional_calibrations = None
if "spatial_calibrations" in data_element:
dimensional_calibrations_list = data_element.get(
"spatial_calibrations")
if len(dimensional_calibrations_list) == len(dimensional_shape):
dimensional_calibrations = list()
for dimension_calibration in dimensional_calibrations_list:
offset = float(dimension_calibration.get("offset", 0.0))
scale = float(dimension_calibration.get("scale", 1.0))
units = dimension_calibration.get("units", "")
units = str(units) if units is not None else str()
if scale != 0.0:
dimensional_calibrations.append(
Calibration.Calibration(offset, scale, units))
else:
dimensional_calibrations.append(Calibration.Calibration())
# intensity calibration
intensity_calibration = None
if "intensity_calibration" in data_element:
intensity_calibration_dict = data_element.get("intensity_calibration")
offset = float(intensity_calibration_dict.get("offset", 0.0))
scale = float(intensity_calibration_dict.get("scale", 1.0))
units = intensity_calibration_dict.get("units", "")
units = str(units) if units is not None else str()
if scale != 0.0:
intensity_calibration = Calibration.Calibration(
offset, scale, units)
# properties (general tags)
metadata = dict()
if "metadata" in data_element:
metadata.update(Utility.clean_dict(data_element.get("metadata")))
if "properties" in data_element and data_element["properties"]:
hardware_source_metadata = metadata.setdefault("hardware_source",
dict())
hardware_source_metadata.update(
Utility.clean_dict(data_element.get("properties")))
# dates are _local_ time and must use this specific ISO 8601 format. 2013-11-17T08:43:21.389391
# time zones are offsets (east of UTC) in the following format "+HHMM" or "-HHMM"
# daylight savings times are time offset (east of UTC) in format "+MM" or "-MM"
# timezone is for conversion and is the Olson timezone string.
# datetime.datetime.strptime(datetime.datetime.isoformat(datetime.datetime.now()), "%Y-%m-%dT%H:%M:%S.%f" )
# datetime_modified, datetime_modified_tz, datetime_modified_dst, datetime_modified_tzname is the time at which this image was modified.
# datetime_original, datetime_original_tz, datetime_original_dst, datetime_original_tzname is the time at which this image was created.
timestamp = data_element.get("timestamp", datetime.datetime.utcnow())
datetime_item = data_element.get(
"datetime_modified",
Utility.get_datetime_item_from_utc_datetime(timestamp))
local_datetime = Utility.get_datetime_from_datetime_item(datetime_item)
dst_value = datetime_item.get("dst", "+00")
tz_value = datetime_item.get("tz", "+0000")
timezone = datetime_item.get("timezone")
time_zone = {"dst": dst_value, "tz": tz_value}
if timezone is not None:
time_zone["timezone"] = timezone
# note: dst is informational only; tz already include dst
tz_adjust = (int(tz_value[1:3]) * 60 +
int(tz_value[3:5])) * (-1 if tz_value[0] == '-' else 1)
utc_datetime = local_datetime - datetime.timedelta(
minutes=tz_adjust) # tz_adjust already contains dst_adjust
timestamp = utc_datetime
return DataAndMetadata.new_data_and_metadata(
data,
intensity_calibration=intensity_calibration,
dimensional_calibrations=dimensional_calibrations,
metadata=metadata,
timestamp=timestamp,
data_descriptor=data_descriptor,
timezone=timezone,
timezone_offset=tz_value)
def function_rgba(
red_data_and_metadata: DataAndMetadata.DataAndMetadata,
green_data_and_metadata: DataAndMetadata.DataAndMetadata,
blue_data_and_metadata: DataAndMetadata.DataAndMetadata,
alpha_data_and_metadata: DataAndMetadata.DataAndMetadata
) -> typing.Optional[DataAndMetadata.DataAndMetadata]:
red_data_and_metadata = DataAndMetadata.promote_ndarray(
red_data_and_metadata)
green_data_and_metadata = DataAndMetadata.promote_ndarray(
green_data_and_metadata)
blue_data_and_metadata = DataAndMetadata.promote_ndarray(
blue_data_and_metadata)
alpha_data_and_metadata = DataAndMetadata.promote_ndarray(
alpha_data_and_metadata)
shape = tuple(
DataAndMetadata.determine_shape(red_data_and_metadata,
green_data_and_metadata,
blue_data_and_metadata,
alpha_data_and_metadata))
red_data_and_metadata = DataAndMetadata.promote_constant(
red_data_and_metadata, shape)
green_data_and_metadata = DataAndMetadata.promote_constant(
green_data_and_metadata, shape)
blue_data_and_metadata = DataAndMetadata.promote_constant(
blue_data_and_metadata, shape)
alpha_data_and_metadata = DataAndMetadata.promote_constant(
alpha_data_and_metadata, shape)
shape = tuple(red_data_and_metadata.data_shape)
def calculate_data():
rgba_image = numpy.empty(shape + (4, ), numpy.uint8)
channels = (blue_data_and_metadata, green_data_and_metadata,
red_data_and_metadata, alpha_data_and_metadata)
for channel_index, channel in enumerate(channels):
data = channel.data
if not Image.is_data_valid(data):
return None
if tuple(data.shape) != shape:
return None
if data.dtype.kind in 'iu':
rgba_image[..., channel_index] = numpy.clip(data, 0, 255)
elif data.dtype.kind in 'f':
rgba_image[..., channel_index] = numpy.clip(
numpy.multiply(data, 255), 0, 255)
else:
return None
return rgba_image
if tuple(green_data_and_metadata.data_shape) != shape or tuple(
blue_data_and_metadata.data_shape) != shape or tuple(
alpha_data_and_metadata.data_shape) != shape:
return None
return DataAndMetadata.new_data_and_metadata(
calculate_data(),
intensity_calibration=red_data_and_metadata.intensity_calibration,
dimensional_calibrations=red_data_and_metadata.dimensional_calibrations
)
def continue_recording(self, current_time: float) -> None:
if self.__recording_state == "recording":
if (current_time - self.__recording_start
) // self.__recording_interval > self.__recording_index - 1:
self.__recording_last = current_time
# first create an empty data item to hold the recorded data if it doesn't already exist
if not self.__recording_data_item:
data_item = DataItem.DataItem(large_format=True)
data_item.ensure_data_source()
data_item.title = _(
"Recording of ") + self.__data_item.title
def process():
self.__document_model.append_data_item(data_item)
display_item = self.__document_controller.document_model.get_display_item_for_data_item(
data_item)
self.__document_controller.show_display_item(
display_item)
return data_item
command = Recorder.RecorderInsertDataItemCommandCommand(
self.__document_controller, self, process)
command.perform()
self.__document_controller.push_undo_command(command)
self.__recording_data_item = data_item
# next grab the current data and stop if it is a sequence (can't record sequences)
current_xdata = self.__last_complete_xdata
if self.__recording_error:
self.__stop_recording()
return
if not current_xdata:
# no first image yet
return
# now record the new data. it may or may not be a new frame at this point.
last_xdata = self.__recording_data_item.xdata
self.__recording_index += 1
if current_xdata and last_xdata and current_xdata.data_shape == self.__recording_data_item.data_shape[
1:]:
# continue, append the new data to existing data and update existing data item
intensity_calibration = last_xdata.intensity_calibration
dimensional_calibrations = last_xdata.dimensional_calibrations
data_descriptor = last_xdata.data_descriptor
sequence_xdata = DataAndMetadata.new_data_and_metadata(
numpy.vstack([last_xdata.data, [current_xdata.data]]),
intensity_calibration=intensity_calibration,
dimensional_calibrations=dimensional_calibrations,
data_descriptor=data_descriptor)
self.__recording_data_item.set_xdata(sequence_xdata)
elif current_xdata and not last_xdata:
# first acquisition, create the sequence
intensity_calibration = current_xdata.intensity_calibration
dimensional_calibrations = [
Calibration.Calibration(
scale=self.__recording_interval, units="s")
] + list(current_xdata.dimensional_calibrations)
data_descriptor = DataAndMetadata.DataDescriptor(
True, current_xdata.data_descriptor.
collection_dimension_count,
current_xdata.data_descriptor.datum_dimension_count)
sequence_xdata = DataAndMetadata.new_data_and_metadata(
current_xdata.data[numpy.newaxis, ...],
intensity_calibration=intensity_calibration,
dimensional_calibrations=dimensional_calibrations,
data_descriptor=data_descriptor)
self.__recording_data_item.set_xdata(sequence_xdata)
self.__recording_transaction = self.__document_model.item_transaction(
self.__recording_data_item)
else:
# something is amiss. stop.
self.__stop_recording()
return
# finally -- check if we've reached the maximum count
if self.__recording_index >= self.__recording_count:
self.__stop_recording()
