def test_dimension_used_for_scale_marker_on_4d_diffraction_image_is_correct(
self):
document_model = DocumentModel.DocumentModel()
document_controller = DocumentController.DocumentController(
self.app.ui, document_model, workspace_id="library")
with contextlib.closing(document_controller):
display_panel = document_controller.selected_display_panel
calibrations = [
Calibration.Calibration(units="y"),
Calibration.Calibration(units="x"),
Calibration.Calibration(units="a"),
Calibration.Calibration(units="b")
]
data_and_metadata = DataAndMetadata.new_data_and_metadata(
numpy.ones((10, 10, 50, 50)),
dimensional_calibrations=calibrations,
data_descriptor=DataAndMetadata.DataDescriptor(False, 2, 2))
data_item = DataItem.new_data_item(data_and_metadata)
document_model.append_data_item(data_item)
display_item = document_model.get_display_item_for_data_item(
data_item)
display_panel.set_display_panel_display_item(display_item)
header_height = display_panel.header_canvas_item.header_height
display_panel.root_container.layout_immediate(
(1000 + header_height, 1000))
# run test
self.assertEqual(
display_panel.display_canvas_item.
_info_overlay_canvas_item_for_test.
_dimension_calibration_for_test.units, "b")
def test_data_timestamp_write_read_round_trip(self):
for version in (3, 4):
s = io.BytesIO()
data_in = numpy.ones((6, 4), numpy.float32)
data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2)
dimensional_calibrations_in = [
Calibration.Calibration(1.1, 2.1, "nm"),
Calibration.Calibration(2, 3, u"µm")
]
intensity_calibration_in = Calibration.Calibration(4.4, 5.5, "six")
metadata_in = dict()
timestamp_in = datetime.datetime(2013, 11, 18, 14, 5, 4, 0)
timezone_in = "America/Los_Angeles"
timezone_offset_in = "-0700"
xdata_in = DataAndMetadata.new_data_and_metadata(
data_in,
data_descriptor=data_descriptor_in,
dimensional_calibrations=dimensional_calibrations_in,
intensity_calibration=intensity_calibration_in,
metadata=metadata_in,
timestamp=timestamp_in,
timezone=timezone_in,
timezone_offset=timezone_offset_in)
dm3_image_utils.save_image(xdata_in, s, version)
s.seek(0)
xdata = dm3_image_utils.load_image(s)
self.assertEqual(timestamp_in, xdata.timestamp)
self.assertEqual(timezone_in, xdata.timezone)
self.assertEqual(timezone_offset_in, xdata.timezone_offset)
def test_build_table_with_two_display_layers_of_same_calibrated_1d_data(
self):
with TestContext.create_memory_context() as test_context:
calibration = Calibration.Calibration(1.0, 2.0, "nm")
document_model = test_context.create_document_model()
data_item = DataItem.DataItem(numpy.array([1.1, 1.2, 1.3, 1.4]))
data_item.set_dimensional_calibration(0, calibration)
data_item.set_intensity_calibration(
Calibration.Calibration(0, 1, "e"))
data_item2 = DataItem.DataItem(numpy.array([2.1, 2.2, 2.3, 2.4]))
data_item2.set_dimensional_calibration(0, calibration)
data_item2.set_intensity_calibration(
Calibration.Calibration(0, 1, "e"))
document_model.append_data_item(data_item)
document_model.append_data_item(data_item2)
display_item = document_model.get_display_item_for_data_item(
data_item)
display_item.append_display_data_channel_for_data_item(data_item2)
display_item._set_display_layer_property(0, "label", "W")
display_item._set_display_layer_property(1, "label", "T")
headers, data = ImportExportManager.build_table(display_item)
self.assertEqual(3, len(headers))
self.assertEqual(3, len(data))
self.assertEqual("X (nm)", headers[0])
self.assertEqual("W (e)", headers[1])
self.assertEqual("T (e)", headers[2])
self.assertTrue(
numpy.array_equal(
calibration.convert_to_calibrated_value(
numpy.arange(0, data_item.data.shape[0])), data[0]))
self.assertTrue(numpy.array_equal(data_item.data, data[1]))
self.assertTrue(numpy.array_equal(data_item2.data, data[2]))
def test_metadata_write_read_round_trip(self):
s = io.BytesIO()
data_in = numpy.ones((6, 4), numpy.float32)
data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2)
dimensional_calibrations_in = [
Calibration.Calibration(1, 2, "nm"),
Calibration.Calibration(2, 3, u"µm")
]
intensity_calibration_in = Calibration.Calibration(4, 5, "six")
metadata_in = {
"abc": 1,
"def": "abc",
"efg": {
"one": 1,
"two": "TWO",
"three": [3, 4, 5]
}
}
xdata_in = DataAndMetadata.new_data_and_metadata(
data_in,
data_descriptor=data_descriptor_in,
dimensional_calibrations=dimensional_calibrations_in,
intensity_calibration=intensity_calibration_in,
metadata=metadata_in)
dm3_image_utils.save_image(xdata_in, s)
s.seek(0)
xdata = dm3_image_utils.load_image(s)
self.assertEqual(metadata_in, xdata.metadata)
def test_rgb_data_write_read_round_trip(self):
for version in (3, 4):
s = io.BytesIO()
data_in = (numpy.random.randn(6, 4, 3) * 255).astype(numpy.uint8)
data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2)
dimensional_calibrations_in = [
Calibration.Calibration(1, 2, "nm"),
Calibration.Calibration(2, 3, u"µm")
]
intensity_calibration_in = Calibration.Calibration(4, 5, "six")
metadata_in = {
"abc": None,
"": "",
"one": [],
"two": {},
"three": [1, None, 2]
}
xdata_in = DataAndMetadata.new_data_and_metadata(
data_in,
data_descriptor=data_descriptor_in,
dimensional_calibrations=dimensional_calibrations_in,
intensity_calibration=intensity_calibration_in,
metadata=metadata_in)
dm3_image_utils.save_image(xdata_in, s, version)
s.seek(0)
xdata = dm3_image_utils.load_image(s)
self.assertTrue(numpy.array_equal(data_in, xdata.data))
self.assertEqual(data_descriptor_in, xdata.data_descriptor)
def test_dimension_used_for_scale_marker_on_2d_data_stack_is_correct(self):
with TestContext.create_memory_context() as test_context:
document_controller = test_context.create_document_controller()
document_model = document_controller.document_model
display_panel = document_controller.selected_display_panel
calibrations = [
Calibration.Calibration(units="s"),
Calibration.Calibration(units="y"),
Calibration.Calibration(units="x")
]
data_and_metadata = DataAndMetadata.new_data_and_metadata(
numpy.ones((50, 10, 10)),
dimensional_calibrations=calibrations,
data_descriptor=DataAndMetadata.DataDescriptor(True, 0, 2))
data_item = DataItem.new_data_item(data_and_metadata)
document_model.append_data_item(data_item)
display_item = document_model.get_display_item_for_data_item(
data_item)
display_panel.set_display_panel_display_item(display_item)
header_height = display_panel.header_canvas_item.header_height
display_panel.root_container.layout_immediate(
(1000 + header_height, 1000))
# run test
self.assertEqual(
display_panel.display_canvas_item.
_info_overlay_canvas_item_for_test.
_dimension_calibration_for_test.units, "x")
def test_data_item_metadata_methods(self):
with create_memory_profile_context() as profile_context:
document_model = DocumentModel.DocumentModel(profile=profile_context.create_profile())
document_controller = self.app.create_document_controller(document_model, "library")
with contextlib.closing(document_controller):
data0 = numpy.arange(64).reshape(8, 8)
data_item = DataItem.DataItem(data0)
data_item.set_intensity_calibration(Calibration.Calibration(0.1, 0.2, "dogs"))
data_item.set_dimensional_calibrations([Calibration.Calibration(0.3, 0.4, "cats"), Calibration.Calibration(0.5, 0.6, "cats")])
metadata = {"title": "Dogs eat cats."}
data_item.metadata = metadata
document_model.append_data_item(data_item)
api = Facade.get_api("~1.0", "~1.0")
library = api.library
data_item_ref = library.data_items[0]
self.assertEqual(data_item_ref.intensity_calibration.units, "dogs")
self.assertEqual(data_item_ref.dimensional_calibrations[1].units, "cats")
self.assertEqual(data_item_ref.metadata, metadata)
data_item_ref.set_intensity_calibration(api.create_calibration(0.11, 0.22, "cats"))
data_item_ref.set_dimensional_calibrations([api.create_calibration(0.33, 0.44, "mice"), api.create_calibration(0.44, 0.66, "mice")])
metadata2 = {"title": "Cats eat mice."}
data_item_ref.set_metadata(metadata2)
self.assertAlmostEqual(data_item.intensity_calibration.offset, 0.11)
self.assertAlmostEqual(data_item.dimensional_calibrations[0].offset, 0.33)
self.assertEqual(data_item.metadata, metadata2)
def test_csv1_exporter_handles_multi_layer_display_item_with_different_calibration(self):
with TestContext.create_memory_context() as test_context:
document_model = test_context.create_document_model()
data_item = DataItem.DataItem(numpy.full((50, ), 0, dtype=numpy.uint32))
data_item.dimensional_calibrations = [Calibration.Calibration(offset=0, scale=1, units="eV")]
data_item.intensity_calibration = Calibration.Calibration(offset=10, scale=2)
data_item2 = DataItem.DataItem(numpy.full((100, ), 1, dtype=numpy.uint32))
data_item2.dimensional_calibrations = [Calibration.Calibration(offset=-10, scale=2, units="eV")]
data_item2.intensity_calibration = Calibration.Calibration(offset=5, scale=2)
document_model.append_data_item(data_item)
document_model.append_data_item(data_item2, False)
display_item = document_model.get_display_item_for_data_item(data_item)
display_item.display_type = "line_plot"
display_item.append_display_data_channel_for_data_item(data_item2)
current_working_directory = os.getcwd()
file_path = os.path.join(current_working_directory, "__file.csv")
handler = ImportExportManager.CSV1ImportExportHandler("csv1-io-handler", "CSV 1D", ["csv"])
handler.write_display_item(None, display_item, file_path, "csv")
self.assertTrue(os.path.exists(file_path))
try:
saved_data = numpy.genfromtxt(file_path, delimiter=", ")
self.assertSequenceEqual(saved_data.shape, (max(data_item.xdata.data_shape[0], data_item2.xdata.data_shape[0]), 4))
self.assertTrue(numpy.allclose(saved_data[:50, 0], numpy.linspace(0, 49, 50)))
self.assertTrue(numpy.allclose(saved_data[:50, 1], 10))
self.assertTrue(numpy.allclose(saved_data[:, 2], numpy.linspace(-10, 188, 100)))
self.assertTrue(numpy.allclose(saved_data[:, 3], 7))
finally:
os.remove(file_path)
def test_display_data_of_3d_data_set_has_correct_shape_and_calibrations(
self):
with TestContext.create_memory_context() as test_context:
document_model = test_context.create_document_model()
intensity_calibration = Calibration.Calibration(units="I")
dim0_calibration = Calibration.Calibration(units="A")
dim1_calibration = Calibration.Calibration(units="B")
dim2_calibration = Calibration.Calibration(units="C")
data_item = DataItem.DataItem(
numpy.zeros((16, 16, 64), numpy.float64))
data_item.set_intensity_calibration(intensity_calibration)
data_item.set_dimensional_calibrations(
[dim0_calibration, dim1_calibration, dim2_calibration])
document_model.append_data_item(data_item)
display_item = document_model.get_display_item_for_data_item(
data_item)
display_data_and_metadata = display_item.display_data_channels[
0].get_calculated_display_values(
True).display_data_and_metadata
self.assertEqual(display_data_and_metadata.dimensional_shape,
(16, 16))
self.assertEqual(display_data_and_metadata.intensity_calibration,
intensity_calibration)
self.assertEqual(
display_data_and_metadata.dimensional_calibrations[0],
dim0_calibration)
self.assertEqual(
display_data_and_metadata.dimensional_calibrations[1],
dim1_calibration)
def test_data_slice_of_sequence_handles_calibrations(self):
data = numpy.zeros((10, 100, 100), dtype=numpy.float32)
intensity_calibration = Calibration.Calibration(0.1, 0.2, "I")
dimensional_calibrations = [
Calibration.Calibration(0.11, 0.22, "S"),
Calibration.Calibration(0.11, 0.22, "A"),
Calibration.Calibration(0.111, 0.222, "B")
]
xdata = DataAndMetadata.new_data_and_metadata(
data,
intensity_calibration=intensity_calibration,
dimensional_calibrations=dimensional_calibrations,
data_descriptor=DataAndMetadata.DataDescriptor(True, 0, 2))
self.assertFalse(xdata[3].is_sequence)
self.assertTrue(xdata[3:4].is_sequence)
self.assertAlmostEqual(xdata[3].intensity_calibration.offset,
xdata.intensity_calibration.offset)
self.assertAlmostEqual(xdata[3].intensity_calibration.scale,
xdata.intensity_calibration.scale)
self.assertEqual(xdata[3].intensity_calibration.units,
xdata.intensity_calibration.units)
self.assertAlmostEqual(xdata[3].dimensional_calibrations[0].offset,
xdata.dimensional_calibrations[1].offset)
self.assertAlmostEqual(xdata[3].dimensional_calibrations[0].scale,
xdata.dimensional_calibrations[1].scale)
self.assertEqual(xdata[3].dimensional_calibrations[0].units,
xdata.dimensional_calibrations[1].units)
self.assertAlmostEqual(xdata[3].dimensional_calibrations[1].offset,
xdata.dimensional_calibrations[2].offset)
self.assertAlmostEqual(xdata[3].dimensional_calibrations[1].scale,
xdata.dimensional_calibrations[2].scale)
self.assertEqual(xdata[3].dimensional_calibrations[1].units,
xdata.dimensional_calibrations[2].units)
def test_signal_type_round_trip(self):
for version in (3, 4):
s = io.BytesIO()
data_in = numpy.ones((12, ), numpy.float32)
data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 1)
dimensional_calibrations_in = [Calibration.Calibration(1, 2, "eV")]
intensity_calibration_in = Calibration.Calibration(4, 5, "e")
metadata_in = {"hardware_source": {"signal_type": "EELS"}}
xdata_in = DataAndMetadata.new_data_and_metadata(
data_in,
data_descriptor=data_descriptor_in,
dimensional_calibrations=dimensional_calibrations_in,
intensity_calibration=intensity_calibration_in,
metadata=metadata_in)
dm3_image_utils.save_image(xdata_in, s, version)
s.seek(0)
xdata = dm3_image_utils.load_image(s)
metadata_expected = {
'hardware_source': {
'signal_type': 'EELS'
},
'Meta Data': {
'Format': 'Spectrum',
'Signal': 'EELS'
}
}
self.assertEqual(metadata_expected, xdata.metadata)
def test_metadata_difficult_types_write_read_round_trip(self):
for version in (3, 4):
s = io.BytesIO()
data_in = numpy.ones((6, 4), numpy.float32)
data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2)
dimensional_calibrations_in = [
Calibration.Calibration(1, 2, "nm"),
Calibration.Calibration(2, 3, u"µm")
]
intensity_calibration_in = Calibration.Calibration(4, 5, "six")
metadata_in = {
"abc": None,
"": "",
"one": [],
"two": {},
"three": [1, None, 2]
}
xdata_in = DataAndMetadata.new_data_and_metadata(
data_in,
data_descriptor=data_descriptor_in,
dimensional_calibrations=dimensional_calibrations_in,
intensity_calibration=intensity_calibration_in,
metadata=metadata_in)
dm3_image_utils.save_image(xdata_in, s, version)
s.seek(0)
xdata = dm3_image_utils.load_image(s)
metadata_expected = {"one": [], "two": {}, "three": [1, 2]}
self.assertEqual(metadata_expected, xdata.metadata)
def get_dimensional_calibrations(self, readout_area: Geometry.IntRect, binning_shape: Geometry.IntSize):
energy_offset_eV = self.instrument.energy_offset_eV
# energy_offset_eV += random.uniform(-1, 1) * self.__energy_per_channel_eV * 5
dimensional_calibrations = [
Calibration.Calibration(),
Calibration.Calibration(offset=energy_offset_eV, scale=self.instrument.energy_per_channel_eV, units="eV")
]
return dimensional_calibrations
def get_dimensional_calibrations(
self, readout_area: typing.Optional[Geometry.IntRect],
binning_shape: typing.Optional[Geometry.IntSize]
) -> typing.Sequence[Calibration.Calibration]:
"""
Subclasses should override this method
"""
return [Calibration.Calibration(), Calibration.Calibration()]
def test_reference_images_load_properly(self):
shape_data_descriptors = (
((3, ), DataAndMetadata.DataDescriptor(False, 0, 1)), # spectrum
((3, 2),
DataAndMetadata.DataDescriptor(False, 1,
1)), # 1d collection of spectra
((3, 4, 5),
DataAndMetadata.DataDescriptor(False, 2,
1)), # 2d collection of spectra
((3, 2), DataAndMetadata.DataDescriptor(True, 0,
1)), # sequence of spectra
((3, 2), DataAndMetadata.DataDescriptor(False, 0, 2)), # image
((4, 3, 2),
DataAndMetadata.DataDescriptor(False, 1,
2)), # 1d collection of images
((3, 4, 5),
DataAndMetadata.DataDescriptor(True, 0, 2)), # sequence of images
)
for (shape, data_descriptor), version in itertools.product(
shape_data_descriptors, (3, 4)):
dimensional_calibrations = list()
for index, dimension in enumerate(shape):
dimensional_calibrations.append(
Calibration.Calibration(1.0 + 0.1 * index,
2.0 + 0.2 * index,
"µ" + "n" * index))
intensity_calibration = Calibration.Calibration(4, 5, "six")
data = numpy.arange(numpy.product(shape),
dtype=numpy.float32).reshape(shape)
name = f"ref_{'T' if data_descriptor.is_sequence else 'F'}_{data_descriptor.collection_dimension_count}_{data_descriptor.datum_dimension_count}.dm{version}"
# import pathlib
# xdata = DataAndMetadata.new_data_and_metadata(data, dimensional_calibrations=dimensional_calibrations, intensity_calibration=intensity_calibration, data_descriptor=data_descriptor)
# file_path = pathlib.Path(__file__).parent / "resources" / name
# with file_path.open('wb') as f:
# dm3_image_utils.save_image(xdata, f, version)
try:
s = io.BytesIO(pkgutil.get_data(__name__, f"resources/{name}"))
xdata = dm3_image_utils.load_image(s)
self.assertAlmostEqual(intensity_calibration.scale,
xdata.intensity_calibration.scale, 6)
self.assertAlmostEqual(intensity_calibration.offset,
xdata.intensity_calibration.offset, 6)
self.assertEqual(intensity_calibration.units,
xdata.intensity_calibration.units)
for c1, c2 in zip(dimensional_calibrations,
xdata.dimensional_calibrations):
self.assertAlmostEqual(c1.scale, c2.scale, 6)
self.assertAlmostEqual(c1.offset, c2.offset, 6)
self.assertEqual(c1.units, c2.units)
self.assertEqual(data_descriptor, xdata.data_descriptor)
self.assertTrue(numpy.array_equal(data, xdata.data))
# print(f"{name} {data_descriptor} PASS")
except Exception as e:
print(f"{name} {data_descriptor} FAIL")
raise
def test_calibration_equality(self):
calibration1 = Calibration.Calibration(1.0, 2.0, "c")
calibration2 = Calibration.Calibration(1.0, 2.0, "c")
self.assertEqual(calibration1, calibration2)
calibration1 = Calibration.Calibration(1.0, 2.1, "c")
calibration2 = Calibration.Calibration(1.0, 2.0, "c")
self.assertNotEqual(calibration1, calibration2)
self.assertNotEqual(calibration1, None)
self.assertTrue(calibration1 != None)
self.assertFalse(calibration1 == None)
def test_metadata_write_read_round_trip(self):
s = io.BytesIO()
data_in = numpy.ones((6, 4), numpy.float32)
dimensional_calibrations_in = [Calibration.Calibration(1, 2, "nm"), Calibration.Calibration(2, 3, u"µm")]
intensity_calibration_in = Calibration.Calibration(4, 5, "six")
metadata_in = {"abc": 1, "def": "abc", "efg": { "one": 1, "two": "TWO", "three": [3, 4, 5] }}
dm3_image_utils.save_image(data_in, dimensional_calibrations_in, intensity_calibration_in, metadata_in, s)
s.seek(0)
data_out, dimensional_calibrations_out, intensity_calibration_out, title_out, metadata_out = dm3_image_utils.load_image(s)
self.assertEqual(metadata_in, metadata_out)
def test_rgb_data_write_read_round_trip(self):
s = io.BytesIO()
data_in = (numpy.random.randn(6, 4, 3) * 255).astype(numpy.uint8)
dimensional_calibrations_in = [Calibration.Calibration(1, 2, "nm"), Calibration.Calibration(2, 3, u"µm")]
intensity_calibration_in = Calibration.Calibration(4, 5, "six")
metadata_in = {"abc": None, "": "", "one": [], "two": {}, "three": [1, None, 2]}
dm3_image_utils.save_image(data_in, dimensional_calibrations_in, intensity_calibration_in, metadata_in, s)
s.seek(0)
data_out, dimensional_calibrations_out, intensity_calibration_out, title_out, metadata_out = dm3_image_utils.load_image(s)
self.assertTrue(numpy.array_equal(data_in, data_out))
def test_metadata_difficult_types_write_read_round_trip(self):
s = io.BytesIO()
data_in = numpy.ones((6, 4), numpy.float32)
dimensional_calibrations_in = [Calibration.Calibration(1, 2, "nm"), Calibration.Calibration(2, 3, u"µm")]
intensity_calibration_in = Calibration.Calibration(4, 5, "six")
metadata_in = {"abc": None, "": "", "one": [], "two": {}, "three": [1, None, 2]}
dm3_image_utils.save_image(data_in, dimensional_calibrations_in, intensity_calibration_in, metadata_in, s)
s.seek(0)
data_out, dimensional_calibrations_out, intensity_calibration_out, title_out, metadata_out = dm3_image_utils.load_image(s)
metadata_expected = {"one": [], "two": {}, "three": [1, 2]}
self.assertEqual(metadata_out, metadata_expected)
def get_dimensional_calibrations(self, readout_area: Geometry.IntRect, binning_shape: Geometry.IntSize):
height = readout_area.height if readout_area else self._sensor_dimensions[0]
width = readout_area.width if readout_area else self._sensor_dimensions[1]
scale_y = self.__tv_pixel_angle
scale_x = self.__tv_pixel_angle
offset_y = -scale_y * height * 0.5
offset_x = -scale_x * width * 0.5
dimensional_calibrations = [
Calibration.Calibration(offset=offset_y, scale=scale_y, units="rad"),
Calibration.Calibration(offset=offset_x, scale=scale_x, units="rad")
]
return dimensional_calibrations
def test_calibrations_write_read_round_trip(self):
s = io.BytesIO()
data_in = numpy.ones((6, 4), numpy.float32)
dimensional_calibrations_in = [Calibration.Calibration(1.1, 2.1, "nm"), Calibration.Calibration(2, 3, u"µm")]
intensity_calibration_in = Calibration.Calibration(4.4, 5.5, "six")
metadata_in = dict()
dm3_image_utils.save_image(data_in, dimensional_calibrations_in, intensity_calibration_in, metadata_in, s)
s.seek(0)
data_out, dimensional_calibrations_out, intensity_calibration_out, title_out, metadata_out = dm3_image_utils.load_image(s)
dimensional_calibrations_out = [Calibration.Calibration(*d) for d in dimensional_calibrations_out]
self.assertEqual(dimensional_calibrations_in, dimensional_calibrations_out)
intensity_calibration_out = Calibration.Calibration(*intensity_calibration_out)
self.assertEqual(intensity_calibration_in, intensity_calibration_out)
def get_dimensional_calibrations(
self, readout_area: typing.Optional[Geometry.IntRect],
binning_shape: typing.Optional[Geometry.IntSize]
) -> typing.Sequence[Calibration.Calibration]:
energy_offset_ev = self.instrument.energy_offset_eV
# energy_offset_ev += random.uniform(-1, 1) * self.__energy_per_channel_eV * 5
dimensional_calibrations = [
Calibration.Calibration(),
Calibration.Calibration(
offset=energy_offset_ev,
scale=self.instrument.energy_per_channel_eV,
units="eV")
]
return dimensional_calibrations
def update(self, data_and_metadata: DataAndMetadata.DataAndMetadata, state: str, scan_shape: Geometry.IntSize, dest_sub_area: Geometry.IntRect, sub_area: Geometry.IntRect, view_id) -> None:
# there are a few techniques for getting data into a data item. this method prefers directly calling the
# document model method update_data_item_partial, which is thread safe. if that method is not available, it
# falls back to the data item method set_data_and_metadata, which must be called from the main thread.
# the hardware source also supplies a data channel which is thread safe and ends up calling set_data_and_metadata
# but we skip that so that the updates fit into this class instead.
frames = self.__multi_acquire_parameters[self.__current_parameters_index]['frames']
sum_frames = self.__multi_acquire_settings['sum_frames']
collection_rank = len(tuple(scan_shape))
data_shape_and_dtype = (tuple(scan_shape) + data_and_metadata.data_shape[collection_rank:], data_and_metadata.data_dtype)
data_descriptor = DataAndMetadata.DataDescriptor(frames > 1 and not sum_frames, collection_rank, len(data_and_metadata.data_shape) - collection_rank)
dimensional_calibrations = data_and_metadata.dimensional_calibrations
if frames > 1 and not sum_frames:
dimensional_calibrations = (Calibration.Calibration(),) + tuple(dimensional_calibrations)
data_shape = data_shape_and_dtype[0]
data_shape = (frames,) + data_shape
data_shape_and_dtype = (data_shape, data_shape_and_dtype[1])
intensity_calibration = data_and_metadata.intensity_calibration
if self.__multi_acquire_settings['use_multi_eels_calibration']:
metadata = data_and_metadata.metadata.get('hardware_source', {})
counts_per_electron = metadata.get('counts_per_electron', 1)
exposure_s = metadata.get('exposure', self.__multi_acquire_parameters[self.__current_parameters_index]['exposure_ms']*0.001)
_number_frames = 1 if not sum_frames else frames
intensity_scale = (data_and_metadata.intensity_calibration.scale / counts_per_electron /
data_and_metadata.dimensional_calibrations[-1].scale / exposure_s / _number_frames)
intensity_calibration = Calibration.Calibration(scale=intensity_scale)
metadata = data_and_metadata.metadata
metadata["MultiAcquire.settings"] = copy.deepcopy(self.__multi_acquire_settings)
metadata["MultiAcquire.parameters"] = copy.deepcopy(self.__multi_acquire_parameters[self.__current_parameters_index])
data_metadata = DataAndMetadata.DataMetadata(data_shape_and_dtype,
intensity_calibration,
dimensional_calibrations,
metadata=data_and_metadata.metadata,
data_descriptor=data_descriptor)
src_slice = sub_area.slice + (Ellipsis,)
dst_slice = dest_sub_area.slice + (Ellipsis,)
if frames > 1:
if sum_frames:
existing_data = self.__data_item.data
if existing_data is not None:
existing_data[dst_slice] += data_and_metadata.data[src_slice]
else:
dst_slice = (self.current_frames_index,) + dst_slice # type: ignore
self.__document_model.update_data_item_partial(self.__data_item, data_metadata, data_and_metadata, src_slice, dst_slice)
self.update_progress(dest_sub_area.bottom)
def test_line_plot_calculates_calibrated_vs_uncalibrated_display_y_values(
self):
with TestContext.create_memory_context() as test_context:
document_controller = test_context.create_document_controller()
document_model = document_controller.document_model
data_item = DataItem.new_data_item(
DataAndMetadata.new_data_and_metadata(
numpy.ones((8, )),
intensity_calibration=Calibration.Calibration(offset=0,
scale=10,
units="nm")))
document_model.append_data_item(data_item)
display_item = document_model.get_display_item_for_data_item(
data_item)
display_item.display_type = "line_plot"
display_panel = document_controller.selected_display_panel
display_panel.set_display_panel_display_item(display_item)
display_panel.display_canvas_item.layout_immediate((640, 480))
self.assertTrue(
numpy.array_equal(
display_panel.display_canvas_item.line_graph_canvas_item.
calibrated_xdata.data, numpy.full((8, ), 10)))
display_item.calibration_style_id = "pixels-top-left"
display_panel.display_canvas_item.layout_immediate((640, 480))
self.assertTrue(
numpy.array_equal(
display_panel.display_canvas_item.line_graph_canvas_item.
calibrated_xdata.data, numpy.ones((8, ))))
def test_line_plot_handles_calibrated_vs_uncalibrated_display(self):
with TestContext.create_memory_context() as test_context:
document_controller = test_context.create_document_controller()
document_model = document_controller.document_model
data_item = DataItem.new_data_item(
DataAndMetadata.new_data_and_metadata(
numpy.ones((8, )),
dimensional_calibrations=[
Calibration.Calibration(offset=0, scale=10, units="nm")
]))
document_model.append_data_item(data_item)
display_item = document_model.get_display_item_for_data_item(
data_item)
display_item.display_type = "line_plot"
display_panel = document_controller.selected_display_panel
display_panel.set_display_panel_display_item(display_item)
display_panel.display_canvas_item.layout_immediate((640, 480))
self.assertEqual(
display_panel.display_canvas_item.line_graph_canvas_item.
calibrated_xdata.dimensional_calibrations[-1].units, "nm")
display_item.calibration_style_id = "pixels-top-left"
display_panel.display_canvas_item.layout_immediate((640, 480))
self.assertFalse(
display_panel.display_canvas_item.line_graph_canvas_item.
calibrated_xdata.dimensional_calibrations[-1].units)
def test_line_plot_handle_calibrated_x_axis_with_negative_scale(self):
with TestContext.create_memory_context() as test_context:
document_controller = test_context.create_document_controller()
document_model = document_controller.document_model
display_panel = document_controller.selected_display_panel
data = numpy.random.randn(100)
data_item = DataItem.DataItem(data)
data_item.set_xdata(
DataAndMetadata.new_data_and_metadata(
data,
dimensional_calibrations=[
Calibration.Calibration(0, -1.0, "e")
]))
document_model.append_data_item(data_item)
display_item = document_model.get_display_item_for_data_item(
data_item)
display_panel.set_display_panel_display_item(display_item)
display_panel.display_canvas_item.layout_immediate((640, 480))
axes = display_panel.display_canvas_item._axes
drawing_context = DrawingContext.DrawingContext()
calibrated_data_min = axes.calibrated_data_min
calibrated_data_max = axes.calibrated_data_max
calibrated_data_range = calibrated_data_max - calibrated_data_min
LineGraphCanvasItem.draw_line_graph(
drawing_context, 480, 640, 0, 0, data_item.xdata,
calibrated_data_min, calibrated_data_range,
axes.calibrated_left_channel, axes.calibrated_right_channel,
axes.x_calibration, "black", "black", None, axes.data_style)
# ensure that the drawing commands are sufficiently populated to have drawn the graph
self.assertGreater(len(drawing_context.commands), 100)
def create_data_item():
nonlocal new_data_item, scan_xdata
if number_frames > 1 and not scan_data_dict['settings'][
'sum_frames']:
data = scan_xdata.data[numpy.newaxis, ...]
dimensional_calibrations = [
Calibration.Calibration()
] + list(scan_xdata.dimensional_calibrations)
data_descriptor = DataAndMetadata.DataDescriptor(
True, scan_xdata.data_descriptor.
collection_dimension_count,
scan_xdata.data_descriptor.datum_dimension_count)
scan_xdata = DataAndMetadata.new_data_and_metadata(
data,
intensity_calibration=scan_xdata.
intensity_calibration,
dimensional_calibrations=dimensional_calibrations,
metadata=scan_xdata.metadata,
timestamp=scan_xdata.timestamp,
data_descriptor=data_descriptor,
timezone=scan_xdata.timezone,
timezone_offset=scan_xdata.timezone_offset)
new_data_item = self.__api.library.create_data_item_from_data_and_metadata(
scan_xdata, title=title)
data_item_ready_event.set()
def test_display_limits_are_reasonable_when_using_calibrated_log_scale(
self):
test_ranges = (((-0.1, 11.0), (0.97, 20.0)), ((1000.0, 1100.0),
(1992.70, 2238.7)),
((1.0, 2.0), (0.09, 20.0)), ((3.0, 4.0), (0.99, 3.0)),
((2.0, 2.1), (0.09, 20.0)), ((0.0, 5.0), (0.98, 10.0)))
for data_in, data_out in test_ranges:
with self.subTest(data_in=data_in, data_out=data_out):
data = numpy.linspace(data_in[0],
data_in[1],
100,
endpoint=False)
intensity_calibration = Calibration.Calibration(-5, 2)
data_style = "log"
calibrated_data_min, calibrated_data_max, y_ticker = LineGraphCanvasItem.calculate_y_axis(
[data], None, None, intensity_calibration, data_style)
axes = LineGraphCanvasItem.LineGraphAxes(
1.0, calibrated_data_min, calibrated_data_max, 0, 100,
None, None, data_style, y_ticker)
self.assertAlmostEqual(axes.uncalibrated_data_min,
data_out[0],
places=1)
self.assertAlmostEqual(axes.uncalibrated_data_max,
data_out[1],
places=1)
calibrated_data = axes.calculate_calibrated_xdata(
DataAndMetadata.new_data_and_metadata(data)).data
assert calibrated_data is not None
data[data <= 0] = numpy.nan
self.assertAlmostEqual(numpy.nanmin(calibrated_data),
math.log10(numpy.nanmin(data)))
self.assertAlmostEqual(numpy.nanmax(calibrated_data),
math.log10(numpy.nanmax(data)))
def __create_spectrum(self) -> DataItem.DataItem:
data = numpy.random.uniform(10, 1000, 1024).astype(numpy.float32)
intensity_calibration = Calibration.Calibration(units="~")
dimensional_calibrations = [
Calibration.Calibration(scale=2.0, units="eV")
]
data_descriptor = DataAndMetadata.DataDescriptor(
is_sequence=False,
collection_dimension_count=0,
datum_dimension_count=1)
xdata = DataAndMetadata.new_data_and_metadata(
data,
intensity_calibration=intensity_calibration,
dimensional_calibrations=dimensional_calibrations,
data_descriptor=data_descriptor)
return DataItem.new_data_item(xdata)
def test_norm(self):
offset_eV = 100
onset_eV = 30
data1 = numpy.zeros((1000, ))
EELSCameraSimulator.plot_norm(data1, 1.0E6, Calibration.Calibration(),
offset_eV, onset_eV)
data2 = numpy.zeros((1000, ))
EELSCameraSimulator.plot_norm(data2, 1.0E6,
Calibration.Calibration(offset=10),
offset_eV, onset_eV)
data3 = numpy.zeros((1000, ))
EELSCameraSimulator.plot_norm(data3, 1.0E6,
Calibration.Calibration(offset=100),
offset_eV, onset_eV)
self.assertEqual(int(data1[500]), int(data2[500 - 10]))
self.assertEqual(int(data1[500]), int(data3[500 - 100]))
