def exec_grab():
# this will execute in a thread; the enclosing async routine will continue when it finishes
try:
start_time = time.time()
max_wait_time = max(
hardware_source.get_current_frame_time() * 1.5, 3)
while hardware_source.is_playing:
if time.time() - start_time > max_wait_time:
success_ref[0] = False
return
time.sleep(0.01)
data_element_groups = hardware_source.get_buffer_data(
-frame_count, frame_count)
for data_element_group in data_element_groups:
if self.cancel_event.is_set():
success_ref[0] = False
break
xdata_group = list()
for data_element in data_element_group:
xdata = ImportExportManager.convert_data_element_to_data_and_metadata(
data_element)
xdata_group.append(xdata)
xdata_group_list.append(xdata_group)
self.progress_model.value = 100
except Exception as e:
import traceback
traceback.print_exc()
success_ref[0] = False
def create_result_data_item(
self, data_dict: typing.Mapping[str, typing.Any]) -> None:
reset_color_cycle()
display_item = None
sorted_indices = numpy.argsort(
[parms['start_ev'] for parms in data_dict['parameter_list']])
display_layer_index = 0
for i in sorted_indices:
index = data_dict['parameter_list'][i]['index']
xdata = ImportExportManager.convert_data_element_to_data_and_metadata(
data_dict['data_element_list'][i])
start_ev = data_dict['parameter_list'][i]['start_ev']
end_ev = data_dict['parameter_list'][i]['end_ev']
number_frames = data_dict['parameter_list'][i]['frames']
exposure_ms = data_dict['parameter_list'][i]['exposure_ms']
summed = ' (summed)' if not data_dict['data_element_list'][i].get(
'is_sequence', False) and number_frames > 1 else ''
data_item = None
if i == sorted_indices[
0] and xdata.datum_dimension_count == 1 and data_dict[
'settings_list'][i]['use_multi_eels_calibration']:
data_item = self.document_controller.library.create_data_item_from_data_and_metadata(
xdata, title='MultiAcquire (stacked)')
display_item = self.__api.library._document_model.get_display_item_for_data_item(
data_item._data_item)
display_layer_index += 1 # display item has one display layer already
#else:
units_str = ' eV' if data_dict['settings_list'][i][
'use_multi_eels_calibration'] else ''
new_data_item = self.document_controller.library.create_data_item_from_data_and_metadata(
xdata,
title='MultiAcquire #{:d}, {:g}-{:g}{}, {:g}x{:g} ms{}'.format(
index + 1, start_ev, end_ev, units_str, number_frames,
exposure_ms, summed))
data_item_metadata = new_data_item.metadata
metadata = dict(
data_item_metadata) if data_item_metadata else dict()
metadata['MultiAcquire.parameters'] = data_dict['parameter_list'][
i]
metadata['MultiAcquire.settings'] = data_dict['settings_list'][i]
new_data_item.set_metadata(metadata)
if display_item:
display_item.append_display_data_channel_for_data_item(
data_item._data_item if data_item else new_data_item.
_data_item)
start_ev = data_dict['parameter_list'][i]['start_ev']
end_ev = data_dict['parameter_list'][i]['end_ev']
display_layer_label = '#{:d}: {:g}-{:g}{}, {:g}x{:g} ms{}'.format(
index + 1, start_ev, end_ev, units_str, number_frames,
exposure_ms, summed)
display_item._set_display_layer_properties(
display_layer_index,
label=display_layer_label,
stroke_color=get_next_color(),
fill_color=None)
display_layer_index += 1
if display_item:
display_item.set_display_property('legend_position', 'top-right')
display_item.title = 'MultiAcquire (stacked)'
show_display_item(self.document_controller, display_item)
def test_data_element_to_extended_data_includes_time_zone(self):
data_element = dict()
data_element["version"] = 1
data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double)
data_element["datetime_modified"] = {'tz': '+0300', 'dst': '+60', 'local_datetime': '2015-06-10T19:31:52.780511'}
xdata = ImportExportManager.convert_data_element_to_data_and_metadata(data_element)
self.assertEqual(xdata.timezone_offset, "+0300")
self.assertEqual(str(xdata.timestamp), "2015-06-10 16:31:52.780511")
def create_result_data_item(self, data_dict):
display_layers = []
reset_color_cycle()
display_item = None
sorted_indices = numpy.argsort(
[parms['start_ev'] for parms in data_dict['parameter_list']])
for i in sorted_indices:
index = data_dict['parameter_list'][i]['index']
xdata = ImportExportManager.convert_data_element_to_data_and_metadata(
data_dict['data_element_list'][i])
start_ev = data_dict['parameter_list'][i]['start_ev']
end_ev = data_dict['parameter_list'][i]['end_ev']
number_frames = data_dict['parameter_list'][i]['frames']
exposure_ms = data_dict['parameter_list'][i]['exposure_ms']
summed = ' (summed)' if not data_dict['data_element_list'][i].get(
'is_sequence', False) and number_frames > 1 else ''
data_item = None
if i == sorted_indices[0] and xdata.datum_dimension_count == 1:
data_item = self.document_controller.library.create_data_item_from_data_and_metadata(
xdata, title='MultiAcquire (stacked)')
display_item = self.__api.library._document_model.get_display_item_for_data_item(
data_item._data_item)
#new_data_item = data_item
#else:
new_data_item = self.document_controller.library.create_data_item_from_data_and_metadata(
xdata,
title='MultiAcquire #{:d}, {:g}-{:g} eV, {:g}x{:g} ms{}'.
format(index + 1, start_ev, end_ev, number_frames, exposure_ms,
summed))
metadata = new_data_item.metadata
metadata['MultiAcquire.parameters'] = data_dict['parameter_list'][
i]
metadata['MultiAcquire.settings'] = data_dict['settings_list'][i]
new_data_item.set_metadata(metadata)
if display_item:
display_item.append_display_data_channel_for_data_item(
data_item._data_item if data_item else new_data_item.
_data_item)
start_ev = data_dict['parameter_list'][i]['start_ev']
end_ev = data_dict['parameter_list'][i]['end_ev']
display_layers.append({
'label':
'#{:d}: {:g}-{:g} eV, {:g}x{:g} ms{}'.format(
index + 1, start_ev, end_ev, number_frames,
exposure_ms, summed),
'data_index':
len(display_layers),
'stroke_color':
get_next_color(),
'fill_color':
None
})
if display_item:
display_item.display_layers = display_layers
display_item.set_display_property('legend_position', 'top-right')
display_item.title = 'MultiAcquire (stacked)'
show_display_item(self.document_controller, display_item)
def grab_sequence(
self, count: int, **kwargs
) -> typing.Optional[typing.List[DataAndMetadata.DataAndMetadata]]:
self.start_playing()
frames = self.acquire_sequence(count)
if frames is not None:
xdatas = list()
for data_element in frames:
data_element["is_sequence"] = True
data_element["collection_dimension_count"] = 0
data_element["datum_dimension_count"] = len(
data_element["data"].shape) - 1
xdata = ImportExportManager.convert_data_element_to_data_and_metadata(
data_element)
xdatas.append(xdata)
return xdatas
return None
def grab_partial(
self,
*,
update_period: float = 1.0) -> typing.Tuple[bool, bool, int]:
# updates the full readout data, returns a tuple of is complete, is canceled, and
# the number of valid rows.
start = self.__start
frame_parameters = self.__camera_frame_parameters
exposure = frame_parameters.exposure_ms / 1000.0
n = min(max(int(update_period / exposure), 1), self.__count - start)
is_complete = start + n == self.__count
# print(f"{start=} {n=} {self.__count=} {is_complete=}")
data_element = self.__camera_device._acquire_sequence((n))
if data_element and not self.__aborted:
xdata = ImportExportManager.convert_data_element_to_data_and_metadata(
data_element)
dimensional_calibrations = tuple(
Calibration.Calibration()
for _ in range(len(self.__collection_shape))) + tuple(
xdata.dimensional_calibrations[1:])
assert self.__xdata
self.__xdata._set_intensity_calibration(
xdata.intensity_calibration)
self.__xdata._set_dimensional_calibrations(
dimensional_calibrations)
if len(self.__collection_shape) > 1:
row_size = self.__collection_shape[-1]
start_row = start // row_size
rows = n // row_size
metadata = dict(xdata.metadata)
metadata.setdefault("hardware_source",
dict())["valid_rows"] = start_row + rows
self.__xdata._set_metadata(metadata)
else:
row_size = 1
# convert from a sequence to a collection.
assert self.__data is not None
self.__data.reshape(
(self.__data.shape[0] * row_size, ) +
self.__data.shape[len(self.__collection_shape):])[
start:start + n, ...] = xdata._data_ex
self.__start = start + n
return is_complete, False, start + n
self.__start = 0
return True, True, 0
def test_data_element_to_extended_data_conversion(self):
data = numpy.ones((8, 6), int)
intensity_calibration = Calibration.Calibration(offset=1, scale=1.1, units="one")
dimensional_calibrations = [Calibration.Calibration(offset=2, scale=2.1, units="two"), Calibration.Calibration(offset=3, scale=2.2, units="two")]
metadata = {"hardware_source": {"one": 1, "two": "b"}}
timestamp = datetime.datetime.now()
data_descriptor = DataAndMetadata.DataDescriptor(is_sequence=False, collection_dimension_count=1, datum_dimension_count=1)
xdata = DataAndMetadata.new_data_and_metadata(data, intensity_calibration=intensity_calibration, dimensional_calibrations=dimensional_calibrations, metadata=metadata, timestamp=timestamp, data_descriptor=data_descriptor)
data_element = ImportExportManager.create_data_element_from_extended_data(xdata)
new_xdata = ImportExportManager.convert_data_element_to_data_and_metadata(data_element)
self.assertTrue(numpy.array_equal(data, new_xdata.data))
self.assertNotEqual(id(new_xdata.intensity_calibration), id(intensity_calibration))
self.assertEqual(new_xdata.intensity_calibration, intensity_calibration)
self.assertNotEqual(id(new_xdata.dimensional_calibrations[0]), id(dimensional_calibrations[0]))
self.assertEqual(new_xdata.dimensional_calibrations, dimensional_calibrations)
self.assertNotEqual(id(new_xdata.metadata), id(metadata))
self.assertEqual(new_xdata.metadata, metadata)
self.assertNotEqual(id(new_xdata.data_descriptor), id(data_descriptor))
self.assertEqual(new_xdata.data_descriptor, data_descriptor)
def acquire_sequence(api: API.API, document_window: API.DocumentWindow,
scan_hardware_source,
camera_hardware_source) -> None:
try:
logging.debug("start")
self.acquisition_state_changed_event.fire({"message": "start"})
try:
camera_hardware_source_id = camera_hardware_source._hardware_source.hardware_source_id
camera_frame_parameters = camera_hardware_source.get_frame_parameters_for_profile_by_index(
0)
if sum_frames:
camera_frame_parameters["processing"] = "sum_project"
scan_frame_parameters = scan_hardware_source.get_frame_parameters_for_profile_by_index(
2)
scan_max_area = 2048 * 2048
scan_param_height = int(scan_frame_parameters["size"][0])
scan_param_width = int(scan_frame_parameters["size"][1])
if scan_param_height * scan_param_width > scan_max_area:
scan_param_height = scan_max_area // scan_param_width
scan_frame_parameters[
"size"] = scan_param_height, scan_param_width
scan_frame_parameters["pixel_time_us"] = int(
1000 * camera_frame_parameters["exposure_ms"] * 0.75)
# long timeout is needed until memory allocation is outside of the acquire_sequence call.
scan_frame_parameters[
"external_clock_wait_time_ms"] = 20000 # int(camera_frame_parameters["exposure_ms"] * 1.5)
scan_frame_parameters["external_clock_mode"] = 1
scan_frame_parameters["ac_line_sync"] = False
scan_frame_parameters["ac_frame_sync"] = False
flyback_pixels = scan_hardware_source._hardware_source.flyback_pixels # using internal API
scan_height = scan_param_height
scan_width = scan_param_width + flyback_pixels
library = document_window.library
camera_hardware_source._hardware_source.set_current_frame_parameters(
camera_hardware_source._hardware_source.
get_frame_parameters_from_dict(
camera_frame_parameters))
camera_hardware_source._hardware_source.acquire_sequence_prepare(
scan_width * scan_height)
with contextlib.closing(
scan_hardware_source.create_record_task(
scan_frame_parameters)) as scan_task:
data_elements = camera_hardware_source._hardware_source.acquire_sequence(
scan_width * scan_height)
data_element = data_elements[0]
# the data_element['data'] ndarray may point to low level memory; we need to get it to disk
# quickly. see note below.
scan_data_list = scan_task.grab()
data_shape = data_element["data"].shape
if flyback_pixels > 0:
data_element["data"] = data_element["data"].reshape(
scan_height, scan_width,
*data_shape[1:])[:,
flyback_pixels:scan_width, :]
else:
data_element["data"] = data_element[
"data"].reshape(scan_height, scan_width,
*data_shape[1:])
if len(scan_data_list) > 0:
collection_calibrations = [
calibration.write_dict() for calibration in
scan_data_list[0].dimensional_calibrations
]
scan_properties = scan_data_list[0].metadata
else:
collection_calibrations = [{}, {}]
scan_properties = {}
if "spatial_calibrations" in data_element:
datum_calibrations = [
copy.deepcopy(spatial_calibration)
for spatial_calibration in
data_element["spatial_calibrations"][1:]
]
else:
datum_calibrations = [{} for i in range(
len(data_element["data"].shape) - 2)]
# combine the dimensional calibrations from the scan data with the datum dimensions calibration from the sequence
data_element["collection_dimension_count"] = 2
data_element[
"spatial_calibrations"] = collection_calibrations + datum_calibrations
data_element.setdefault(
"metadata",
dict())["scan_detector"] = scan_properties.get(
"hardware_source", dict())
data_and_metadata = ImportExportManager.convert_data_element_to_data_and_metadata(
data_element)
def create_and_display_data_item():
data_item = library.get_data_item_for_hardware_source(
scan_hardware_source,
channel_id=camera_hardware_source_id,
processor_id="summed",
create_if_needed=True,
large_format=True)
data_item.title = _("Spectrum Image {}".format(
" x ".join([
str(d) for d in
data_and_metadata.dimensional_shape
])))
# the data item should not have any other 'clients' at this point; so setting the
# data and metadata will immediately unload the data (and write to disk). this is important,
# because the data (up to this point) can be shared data from the DLL.
data_item.set_data_and_metadata(data_and_metadata)
# assert not data_item._data_item.is_data_loaded
# now to display it will reload the data (presumably from an HDF5 or similar on-demand format).
document_window.display_data_item(data_item)
for scan_data_and_metadata in scan_data_list:
scan_channel_id = scan_data_and_metadata.metadata[
"hardware_source"]["channel_id"]
scan_channel_name = scan_data_and_metadata.metadata[
"hardware_source"]["channel_name"]
channel_id = camera_hardware_source_id + "_" + scan_channel_id
data_item = library.get_data_item_for_hardware_source(
scan_hardware_source,
channel_id=channel_id,
create_if_needed=True)
data_item.title = "{} ({})".format(
_("Spectrum Image"), scan_channel_name)
data_item.set_data_and_metadata(
scan_data_and_metadata)
document_window.display_data_item(data_item)
document_window.queue_task(create_and_display_data_item
) # must occur on UI thread
finally:
self.acquisition_state_changed_event.fire(
{"message": "end"})
logging.debug("end")
except Exception as e:
import traceback
traceback.print_exc()
def process_display_queue(self):
while True:
try:
data_dict = self.__display_queue.get(timeout=1)
except queue.Empty:
if self.__data_processed_event.is_set(
) and not self.__acquisition_running:
self.__close_data_item_refs()
break
else:
index = data_dict['parameters']['index']
line_number = data_dict['parameters']['line_number']
data_element = data_dict['data_element']
line_data = data_element['data']
start_ev = data_dict['parameters']['start_ev']
end_ev = data_dict['parameters']['end_ev']
number_frames = data_dict['parameters']['frames']
exposure_ms = data_dict['parameters']['exposure_ms']
print('got data from display queue')
if not self.result_data_items.get(index):
print('creating new data item')
spatial_calibrations = data_element['spatial_calibrations']
data_element['data'] = line_data[numpy.newaxis, ...]
data_element['collection_dimension_count'] = 2
data_element[
'spatial_calibrations'] = self.multi_acquire_controller.scan_calibrations[
0:1] + spatial_calibrations
metadata = data_element.get('metadata', {})
metadata['MultiAcquire.parameters'] = data_dict[
'parameters']
metadata['MultiAcquire.settings'] = data_dict['settings']
data_element['metadata'] = data_dict['parameters']
new_xdata = ImportExportManager.convert_data_element_to_data_and_metadata(
data_element)
title = ('MultiAcquire (stitched)'
if data_dict.get('stitched_data') else
'MultiAcquire #{:d}, {:g}-{:g} eV, {:g}x{:g} ms'.
format(index + 1, start_ev, end_ev, number_frames,
exposure_ms))
data_item_ready_event = threading.Event()
new_data_item = None
def create_data_item():
nonlocal new_data_item
# we have to create the initial data item with some data that has more than 2 dimensions
# otherwise Swift does not use HDF5 and we will have a problem if it grows too big later
new_data_item = self.__api.library.create_data_item_from_data_and_metadata(
new_xdata, title=title)
data_item_ready_event.set()
self.__api.queue_task(create_data_item)
data_item_ready_event.wait()
number_lines = data_dict['parameters'].get(
'number_lines',
self.multi_acquire_controller.number_lines)
max_shape = (number_lines, ) + line_data.shape
new_appendable_data_item = AppendableDataItemFixedSize(
new_data_item, max_shape, self.__api)
new_appendable_data_item.enter_write_suspend_state()
self.result_data_items[index] = new_appendable_data_item
# add the line we have already in our data item to the appendable data item to have them in the
# same state
self.result_data_items[index].add_data((line_number, ...),
line_data)
self.__api.queue_task(lambda: self.result_data_items[
index].get_partial_data_item(
(slice(0, line_number + 1), ...)))
del new_xdata
del new_appendable_data_item
else:
self.result_data_items[index].add_data((line_number, ...),
line_data)
self.__api.queue_task(lambda: self.result_data_items[
index].get_partial_data_item(
(slice(0, line_number + 1), ...)))
del line_data
del data_element
del data_dict
self.__display_queue.task_done()
print('displayed line {:.0f}'.format(line_number))
