def test_importing_rgb_does_not_set_large_format(self):
data_item = DataItem.DataItem(numpy.zeros((8, 8, 4), dtype=float))
with contextlib.closing(data_item):
data_item_rgb = DataItem.DataItem(numpy.zeros((8, 8, 4), dtype=numpy.uint8))
with contextlib.closing(data_item_rgb):
data_element = ImportExportManager.create_data_element_from_data_item(data_item, include_data=True)
data_element_rgb = ImportExportManager.create_data_element_from_data_item(data_item_rgb, include_data=True)
data_element.pop("large_format")
data_element_rgb.pop("large_format")
with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item:
with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element_rgb)) as data_item_rgb:
self.assertTrue(data_item.large_format)
self.assertFalse(data_item_rgb.large_format)
def test_time_zone_in_extended_data_to_data_element_to_data_item_conversion(
self):
# test the whole path, redundant?
data = numpy.ones((8, 6), numpy.int)
metadata = {
"description": {
"time_zone": {
"tz": "+0300",
"dst": "+60"
}
},
"hardware_source": {
"one": 1,
"two": "b"
}
}
timestamp = datetime.datetime(2013, 11, 18, 14, 5, 4, 1)
xdata = DataAndMetadata.new_data_and_metadata(data,
metadata=metadata,
timestamp=timestamp)
data_element = ImportExportManager.create_data_element_from_extended_data(
xdata)
data_item = ImportExportManager.create_data_item_from_data_element(
data_element)
self.assertEqual(data_item.metadata["description"]["time_zone"]["tz"],
"+0300")
self.assertEqual(data_item.metadata["description"]["time_zone"]["dst"],
"+60")
self.assertEqual("2013-11-18 14:05:04.000001", str(data_item.created))
def test_data_element_date_gets_set_as_data_item_created_date(self):
data_element = dict()
data_element["version"] = 1
data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double)
data_element["datetime_modified"] = {
'tz': '+0300',
'dst': '+60',
'local_datetime': '2015-06-10T19:31:52.780511'
}
data_item = ImportExportManager.create_data_item_from_data_element(
data_element)
self.assertIsNotNone(data_item.created)
self.assertEqual(data_item.timezone_offset, "+0300")
local_offset_seconds = int(
round((datetime.datetime.now() -
datetime.datetime.utcnow()).total_seconds()))
# check both matches for DST
match1 = datetime.datetime(year=2015,
month=6,
day=10,
hour=19 - 3,
minute=31,
second=52,
microsecond=780511) + datetime.timedelta(
seconds=local_offset_seconds)
match2 = datetime.datetime(year=2015,
month=6,
day=10,
hour=19 - 3,
minute=31,
second=52,
microsecond=780511) + datetime.timedelta(
seconds=local_offset_seconds + 3600)
self.assertTrue(data_item.created_local == match1
or data_item.created_local == match2)
def acquire_stack_and_sum(number_frames, energy_offset_per_frame,
document_controller, final_layout_fn):
# grab the document model and workspace for convenience
with document_controller.create_task_context_manager(
_("Multiple Shift EELS Acquire"), "table") as task:
# acquire the stack. it will be added to the document by queueing to the main thread at the end of this method.
stack_data_item = acquire_series(number_frames,
energy_offset_per_frame, task)
stack_data_item.title = _("Spectrum Stack")
# align and sum the stack
data_element = dict()
summed_image, shifts = align_stack(stack_data_item.data, task)
# add the summed image to Swift
data_element["data"] = summed_image
data_element["title"] = "Aligned and summed spectra"
# strip off the first dimension that we sum over
for dimensional_calibration in stack_data_item.dimensional_calibrations[
1:]:
data_element.setdefault(
"spatial_calibrations", list()).append({
"origin":
dimensional_calibration.offset, # TODO: fix me
"scale":
dimensional_calibration.scale,
"units":
dimensional_calibration.units
})
# set the energy dispersive calibration so that the ZLP is at zero eV
zlp_position_pixels = numpy.sum(summed_image, axis=0).argmax()
zlp_position_calibrated_units = -zlp_position_pixels * data_element[
"spatial_calibrations"][1]["scale"]
data_element["spatial_calibrations"][1][
"offset"] = zlp_position_calibrated_units
sum_data_item = ImportExportManager.create_data_item_from_data_element(
data_element)
dispersive_sum = numpy.sum(summed_image, axis=1)
differential = numpy.diff(dispersive_sum)
top = numpy.argmax(differential)
bottom = numpy.argmin(differential)
_midpoint = numpy.mean([bottom, top]) / dispersive_sum.shape[0]
_integration_width = float(
numpy.abs(bottom - top)) / dispersive_sum.shape[
0] #* data_element["spatial_calibrations"][0]["scale"]
document_controller.queue_task(final_layout_fn)
document_controller.queue_task(
functools.partial(show_in_panel, stack_data_item,
document_controller,
"multiple_shift_eels_stack"))
document_controller.queue_task(
functools.partial(
show_in_panel, sum_data_item, document_controller,
"multiple_shift_eels_aligned_summed_stack"))
document_controller.queue_task(
functools.partial(add_line_profile, sum_data_item,
document_controller,
"multiple_shift_eels_spectrum",
_midpoint, _integration_width))
def test_data_item_to_data_element_and_back_keeps_large_format_flag(self):
data_item = DataItem.DataItem(numpy.zeros((4, 4, 4)), large_format=True)
with contextlib.closing(data_item):
data_element = ImportExportManager.create_data_element_from_data_item(data_item, include_data=True)
self.assertTrue(data_element.get("large_format"))
with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item:
self.assertTrue(data_item.large_format)
def test_data_element_to_data_item_includes_time_zone(self):
data_element = dict()
data_element["version"] = 1
data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double)
data_element["datetime_modified"] = {'tz': '+0300', 'dst': '+60', 'local_datetime': '2015-06-10T19:31:52.780511'}
with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item:
self.assertEqual(data_item.timezone_offset, "+0300")
self.assertEqual(str(data_item.created), "2015-06-10 16:31:52.780511")
def test_convert_data_element_records_time_zone_in_data_item_metadata(self):
data_element = dict()
data_element["version"] = 1
data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double)
data_element["datetime_modified"] = Utility.get_current_datetime_item()
with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item:
self.assertIsNotNone(data_item.created)
self.assertEqual(data_item.timezone_offset, data_element["datetime_modified"]["tz"])
def test_data_element_with_uuid_assigns_uuid_to_data_item(self):
data_element = dict()
data_element["version"] = 1
data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double)
data_element_uuid = uuid.uuid4()
data_element["uuid"] = str(data_element_uuid)
with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item:
self.assertEqual(data_item.uuid, data_element_uuid)
def test_convert_data_element_sets_timezone_and_timezone_offset_if_present(self):
data_element = dict()
data_element["version"] = 1
data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double)
data_element["datetime_modified"] = {'tz': '+0300', 'dst': '+60', 'local_datetime': '2015-06-10T19:31:52.780511', 'timezone': 'Europe/Athens'}
with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item:
self.assertIsNotNone(data_item.created)
self.assertEqual(data_item.timezone, "Europe/Athens")
self.assertEqual(data_item.timezone_offset, "+0300")
def test_creating_data_element_with_sequence_and_implicit_datum_size_data_makes_correct_data_item(self):
data_element = dict()
data_element["version"] = 1
data_element["data"] = numpy.zeros((4, 16, 16), dtype=numpy.double)
data_element["is_sequence"] = True
with contextlib.closing(ImportExportManager.create_data_item_from_data_element(data_element)) as data_item:
self.assertEqual(data_item.is_sequence, True)
self.assertEqual(data_item.collection_dimension_count, 0)
self.assertEqual(data_item.datum_dimension_count, 2)
self.assertEqual(data_item.xdata.is_sequence, True)
self.assertEqual(data_item.xdata.collection_dimension_count, 0)
self.assertEqual(data_item.xdata.datum_dimension_count, 2)
def test_creating_data_element_with_sequence_data_makes_correct_data_item(self):
data_element = dict()
data_element["version"] = 1
data_element["data"] = numpy.zeros((4, 16, 16), dtype=numpy.double)
data_element["is_sequence"] = True
data_element["collection_dimension_count"] = 0
data_element["datum_dimension_count"] = 2
data_item = ImportExportManager.create_data_item_from_data_element(data_element)
self.assertEqual(data_item.is_sequence, True)
self.assertEqual(data_item.collection_dimension_count, 0)
self.assertEqual(data_item.datum_dimension_count, 2)
self.assertEqual(data_item.xdata.is_sequence, True)
self.assertEqual(data_item.xdata.collection_dimension_count, 0)
self.assertEqual(data_item.xdata.datum_dimension_count, 2)
def test_sub_area_size_change(self):
with TestContext.create_memory_context() as test_context:
document_model = test_context.create_document_model()
data_element = dict()
data_element["version"] = 1
data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double)
data_item = ImportExportManager.create_data_item_from_data_element(data_element)
document_model.append_data_item(data_item)
self.assertEqual(data_item.dimensional_shape, (16, 16))
self.assertEqual(data_item.data_dtype, numpy.double)
data_element["data"] = numpy.zeros((8, 8), dtype=numpy.double)
data_element["sub_area"] = ((0,0), (4, 8))
ImportExportManager.update_data_item_from_data_element(data_item, data_element)
self.assertEqual(data_item.dimensional_shape, (8, 8))
self.assertEqual(data_item.data_dtype, numpy.double)
data_element["data"] = numpy.zeros((8, 8), dtype=float)
data_element["sub_area"] = ((0,0), (4, 8))
ImportExportManager.update_data_item_from_data_element(data_item, data_element)
self.assertEqual(data_item.dimensional_shape, (8, 8))
self.assertEqual(data_item.data_dtype, float)
def test_sub_area_size_change(self):
document_model = DocumentModel.DocumentModel()
with contextlib.closing(document_model):
data_element = dict()
data_element["version"] = 1
data_element["data"] = numpy.zeros((16, 16), dtype=numpy.double)
data_item = ImportExportManager.create_data_item_from_data_element(data_element)
document_model.append_data_item(data_item)
self.assertEqual(data_item.dimensional_shape, (16, 16))
self.assertEqual(data_item.data_dtype, numpy.double)
data_element["data"] = numpy.zeros((8, 8), dtype=numpy.double)
data_element["sub_area"] = ((0,0), (4, 8))
ImportExportManager.update_data_item_from_data_element(data_item, data_element)
self.assertEqual(data_item.dimensional_shape, (8, 8))
self.assertEqual(data_item.data_dtype, numpy.double)
data_element["data"] = numpy.zeros((8, 8), dtype=numpy.float)
data_element["sub_area"] = ((0,0), (4, 8))
ImportExportManager.update_data_item_from_data_element(data_item, data_element)
self.assertEqual(data_item.dimensional_shape, (8, 8))
self.assertEqual(data_item.data_dtype, numpy.float)
def acquire_stack_and_sum(
number_frames: int, energy_offset: float,
dark_ref_enabled: bool,
dark_ref_data: typing.Optional[_NDArray], cross_cor: bool,
document_controller: DocumentController.DocumentController,
final_layout_fn: typing.Callable[[], None]) -> None:
# grab the document model and workspace for convenience
with document_controller.create_task_context_manager(
_("Multiple Shift EELS Acquire"), "table") as task:
# acquire the stack. it will be added to the document by
# queueing to the main thread at the end of this method.
stack_data_item = acquire_series(number_frames, energy_offset,
dark_ref_enabled,
dark_ref_data, task)
stack_data_item.title = _("Spectrum Stack")
# align and sum the stack
data_element: typing.Dict[str, typing.Any] = dict()
stack_data = stack_data_item.data
if stack_data is not None:
if cross_cor:
# Apply cross-correlation between subsequent acquired
# images and align the image stack
summed_image, _1 = align_stack(stack_data, task)
else:
# If user does not desire the cross-correlation to happen
# then simply sum the stack (eg, when acquiring dark data)
summed_image = numpy.sum(stack_data, axis=0)
# add the summed image to Swift
data_element["data"] = summed_image
data_element["title"] = "Aligned and summed spectra"
# strip off the first dimension that we sum over
for dimensional_calibration in (
stack_data_item.dimensional_calibrations[1:]):
data_element.setdefault(
"spatial_calibrations", list()).append({
"origin":
dimensional_calibration.offset, # TODO: fix me
"scale":
dimensional_calibration.scale,
"units":
dimensional_calibration.units
})
# set the energy dispersive calibration so that the ZLP is at
# zero eV
zlp_position_pixels = numpy.sum(summed_image, axis=0).argmax()
zlp_position_calibrated_units = (
-zlp_position_pixels *
data_element["spatial_calibrations"][1]["scale"])
data_element["spatial_calibrations"][1]["offset"] = (
zlp_position_calibrated_units)
sum_data_item = (
ImportExportManager.create_data_item_from_data_element(
data_element))
dispersive_sum = numpy.sum(summed_image, axis=1)
differential = numpy.diff(dispersive_sum) # type: ignore
top = numpy.argmax(differential)
bottom = numpy.argmin(differential)
_midpoint = numpy.mean([bottom, top]) / dispersive_sum.shape[0]
_integration_width = (float(numpy.abs(bottom - top)) /
dispersive_sum.shape[0])
document_controller.queue_task(final_layout_fn)
document_controller.queue_task(
functools.partial(show_in_panel, stack_data_item,
document_controller,
"multiple_shift_eels_stack"))
document_controller.queue_task(
functools.partial(
show_in_panel, sum_data_item, document_controller,
"multiple_shift_eels_aligned_summed_stack"))
document_controller.queue_task(
functools.partial(add_line_profile, sum_data_item,
document_controller,
"multiple_shift_eels_spectrum",
_midpoint, _integration_width))
