def test_signal_type_round_trip(self): for version in (3, 4): s = io.BytesIO() data_in = numpy.ones((12, ), numpy.float32) data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 1) dimensional_calibrations_in = [Calibration.Calibration(1, 2, "eV")] intensity_calibration_in = Calibration.Calibration(4, 5, "e") metadata_in = {"hardware_source": {"signal_type": "EELS"}} xdata_in = DataAndMetadata.new_data_and_metadata( data_in, data_descriptor=data_descriptor_in, dimensional_calibrations=dimensional_calibrations_in, intensity_calibration=intensity_calibration_in, metadata=metadata_in) dm3_image_utils.save_image(xdata_in, s, version) s.seek(0) xdata = dm3_image_utils.load_image(s) metadata_expected = { 'hardware_source': { 'signal_type': 'EELS' }, 'Meta Data': { 'Format': 'Spectrum', 'Signal': 'EELS' } } self.assertEqual(metadata_expected, xdata.metadata)
def test_rgb_data_write_read_round_trip(self): for version in (3, 4): s = io.BytesIO() data_in = (numpy.random.randn(6, 4, 3) * 255).astype(numpy.uint8) data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2) dimensional_calibrations_in = [ Calibration.Calibration(1, 2, "nm"), Calibration.Calibration(2, 3, u"µm") ] intensity_calibration_in = Calibration.Calibration(4, 5, "six") metadata_in = { "abc": None, "": "", "one": [], "two": {}, "three": [1, None, 2] } xdata_in = DataAndMetadata.new_data_and_metadata( data_in, data_descriptor=data_descriptor_in, dimensional_calibrations=dimensional_calibrations_in, intensity_calibration=intensity_calibration_in, metadata=metadata_in) dm3_image_utils.save_image(xdata_in, s, version) s.seek(0) xdata = dm3_image_utils.load_image(s) self.assertTrue(numpy.array_equal(data_in, xdata.data)) self.assertEqual(data_descriptor_in, xdata.data_descriptor)
def test_metadata_difficult_types_write_read_round_trip(self): for version in (3, 4): s = io.BytesIO() data_in = numpy.ones((6, 4), numpy.float32) data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2) dimensional_calibrations_in = [ Calibration.Calibration(1, 2, "nm"), Calibration.Calibration(2, 3, u"µm") ] intensity_calibration_in = Calibration.Calibration(4, 5, "six") metadata_in = { "abc": None, "": "", "one": [], "two": {}, "three": [1, None, 2] } xdata_in = DataAndMetadata.new_data_and_metadata( data_in, data_descriptor=data_descriptor_in, dimensional_calibrations=dimensional_calibrations_in, intensity_calibration=intensity_calibration_in, metadata=metadata_in) dm3_image_utils.save_image(xdata_in, s, version) s.seek(0) xdata = dm3_image_utils.load_image(s) metadata_expected = {"one": [], "two": {}, "three": [1, 2]} self.assertEqual(metadata_expected, xdata.metadata)
def test_data_timestamp_write_read_round_trip(self): for version in (3, 4): s = io.BytesIO() data_in = numpy.ones((6, 4), numpy.float32) data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2) dimensional_calibrations_in = [ Calibration.Calibration(1.1, 2.1, "nm"), Calibration.Calibration(2, 3, u"µm") ] intensity_calibration_in = Calibration.Calibration(4.4, 5.5, "six") metadata_in = dict() timestamp_in = datetime.datetime(2013, 11, 18, 14, 5, 4, 0) timezone_in = "America/Los_Angeles" timezone_offset_in = "-0700" xdata_in = DataAndMetadata.new_data_and_metadata( data_in, data_descriptor=data_descriptor_in, dimensional_calibrations=dimensional_calibrations_in, intensity_calibration=intensity_calibration_in, metadata=metadata_in, timestamp=timestamp_in, timezone=timezone_in, timezone_offset=timezone_offset_in) dm3_image_utils.save_image(xdata_in, s, version) s.seek(0) xdata = dm3_image_utils.load_image(s) self.assertEqual(timestamp_in, xdata.timestamp) self.assertEqual(timezone_in, xdata.timezone) self.assertEqual(timezone_offset_in, xdata.timezone_offset)
def test_metadata_write_read_round_trip(self): s = io.BytesIO() data_in = numpy.ones((6, 4), numpy.float32) data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2) dimensional_calibrations_in = [ Calibration.Calibration(1, 2, "nm"), Calibration.Calibration(2, 3, u"µm") ] intensity_calibration_in = Calibration.Calibration(4, 5, "six") metadata_in = { "abc": 1, "def": "abc", "efg": { "one": 1, "two": "TWO", "three": [3, 4, 5] } } xdata_in = DataAndMetadata.new_data_and_metadata( data_in, data_descriptor=data_descriptor_in, dimensional_calibrations=dimensional_calibrations_in, intensity_calibration=intensity_calibration_in, metadata=metadata_in) dm3_image_utils.save_image(xdata_in, s) s.seek(0) xdata = dm3_image_utils.load_image(s) self.assertEqual(metadata_in, xdata.metadata)
def test_reference_images_load_properly(self): shape_data_descriptors = ( ((3, ), DataAndMetadata.DataDescriptor(False, 0, 1)), # spectrum ((3, 2), DataAndMetadata.DataDescriptor(False, 1, 1)), # 1d collection of spectra ((3, 4, 5), DataAndMetadata.DataDescriptor(False, 2, 1)), # 2d collection of spectra ((3, 2), DataAndMetadata.DataDescriptor(True, 0, 1)), # sequence of spectra ((3, 2), DataAndMetadata.DataDescriptor(False, 0, 2)), # image ((4, 3, 2), DataAndMetadata.DataDescriptor(False, 1, 2)), # 1d collection of images ((3, 4, 5), DataAndMetadata.DataDescriptor(True, 0, 2)), # sequence of images ) for (shape, data_descriptor), version in itertools.product( shape_data_descriptors, (3, 4)): dimensional_calibrations = list() for index, dimension in enumerate(shape): dimensional_calibrations.append( Calibration.Calibration(1.0 + 0.1 * index, 2.0 + 0.2 * index, "µ" + "n" * index)) intensity_calibration = Calibration.Calibration(4, 5, "six") data = numpy.arange(numpy.product(shape), dtype=numpy.float32).reshape(shape) name = f"ref_{'T' if data_descriptor.is_sequence else 'F'}_{data_descriptor.collection_dimension_count}_{data_descriptor.datum_dimension_count}.dm{version}" # import pathlib # xdata = DataAndMetadata.new_data_and_metadata(data, dimensional_calibrations=dimensional_calibrations, intensity_calibration=intensity_calibration, data_descriptor=data_descriptor) # file_path = pathlib.Path(__file__).parent / "resources" / name # with file_path.open('wb') as f: # dm3_image_utils.save_image(xdata, f, version) try: s = io.BytesIO(pkgutil.get_data(__name__, f"resources/{name}")) xdata = dm3_image_utils.load_image(s) self.assertAlmostEqual(intensity_calibration.scale, xdata.intensity_calibration.scale, 6) self.assertAlmostEqual(intensity_calibration.offset, xdata.intensity_calibration.offset, 6) self.assertEqual(intensity_calibration.units, xdata.intensity_calibration.units) for c1, c2 in zip(dimensional_calibrations, xdata.dimensional_calibrations): self.assertAlmostEqual(c1.scale, c2.scale, 6) self.assertAlmostEqual(c1.offset, c2.offset, 6) self.assertEqual(c1.units, c2.units) self.assertEqual(data_descriptor, xdata.data_descriptor) self.assertTrue(numpy.array_equal(data, xdata.data)) # print(f"{name} {data_descriptor} PASS") except Exception as e: print(f"{name} {data_descriptor} FAIL") raise
def test_metadata_export_large_integer(self): s = io.BytesIO() data_in = numpy.ones((6, 4), numpy.float32) data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2) dimensional_calibrations_in = [ Calibration.Calibration(1, 2, "nm"), Calibration.Calibration(2, 3, u"µm") ] intensity_calibration_in = Calibration.Calibration(4, 5, "six") metadata_in = {"abc": 999999999999} xdata_in = DataAndMetadata.new_data_and_metadata( data_in, data_descriptor=data_descriptor_in, dimensional_calibrations=dimensional_calibrations_in, intensity_calibration=intensity_calibration_in, metadata=metadata_in) dm3_image_utils.save_image(xdata_in, s) s.seek(0) xdata = dm3_image_utils.load_image(s) metadata_expected = {"abc": 999999999999} self.assertEqual(metadata_expected, xdata.metadata)
def test_metadata_write_read_round_trip(self): for version in (3, 4): s = io.BytesIO() data_in = numpy.ones((6, 4), numpy.float32) data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2) dimensional_calibrations_in = [ Calibration.Calibration(1, 2, "nm"), Calibration.Calibration(2, 3, u"µm") ] intensity_calibration_in = Calibration.Calibration(4, 5, "six") metadata_in = { "abc": 1, "def": "abc", "efg": { "one": 1, "two": "TWO", "three": [3, 4, 5], "threef": [3.0, 4.0, 5.0], "four": (32, 32), "fourf": (33.0, 34.0), "six": ((1, 2), (3, 4)), "sixf": ((1.0, 2.0), (3.0, 4.0)), "seven": [[1, 2], [3, 4]], "sevenf": [[1.0, 2.0], [3.0, 4.0]], # the following will not work until there is a schema or other type hinting to distinguish # this from the "six" case. # "eight": (1, 2, 3, 4), "eightf": (1.0, 2.0, 3.0, 4.0), } } xdata_in = DataAndMetadata.new_data_and_metadata( data_in, data_descriptor=data_descriptor_in, dimensional_calibrations=dimensional_calibrations_in, intensity_calibration=intensity_calibration_in, metadata=metadata_in) dm3_image_utils.save_image(xdata_in, s, version) s.seek(0) xdata = dm3_image_utils.load_image(s) self.assertEqual(metadata_in, xdata.metadata)
def test_calibrations_write_read_round_trip(self): s = io.BytesIO() data_in = numpy.ones((6, 4), numpy.float32) data_descriptor_in = DataAndMetadata.DataDescriptor(False, 0, 2) dimensional_calibrations_in = [ Calibration.Calibration(1.1, 2.1, "nm"), Calibration.Calibration(2, 3, u"µm") ] intensity_calibration_in = Calibration.Calibration(4.4, 5.5, "six") metadata_in = dict() xdata_in = DataAndMetadata.new_data_and_metadata( data_in, data_descriptor=data_descriptor_in, dimensional_calibrations=dimensional_calibrations_in, intensity_calibration=intensity_calibration_in, metadata=metadata_in) dm3_image_utils.save_image(xdata_in, s) s.seek(0) xdata = dm3_image_utils.load_image(s) self.assertEqual(dimensional_calibrations_in, xdata.dimensional_calibrations) self.assertEqual(intensity_calibration_in, xdata.intensity_calibration)
def disabled_test_specific_file(self): file_path = "/path/to/test.dm3" with open(file_path, "rb") as f: xdata = dm3_image_utils.load_image(f)
def test_data_write_read_round_trip(self): def db_make_directory_if_needed(directory_path): if os.path.exists(directory_path): if not os.path.isdir(directory_path): raise OSError("Path is not a directory:", directory_path) else: os.makedirs(directory_path) class numpy_array_type: def __init__(self, shape, dtype): self.data = numpy.ones(shape, dtype) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): pass class h5py_array_type: def __init__(self, shape, dtype): current_working_directory = os.getcwd() self.__workspace_dir = os.path.join(current_working_directory, "__Test") db_make_directory_if_needed(self.__workspace_dir) self.f = h5py.File( os.path.join(self.__workspace_dir, "file.h5"), "a") self.data = self.f.create_dataset("data", data=numpy.ones( shape, dtype)) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self.f.close() shutil.rmtree(self.__workspace_dir) array_types = numpy_array_type, h5py_array_type dtypes = (numpy.float32, numpy.float64, numpy.complex64, numpy.complex128, numpy.int16, numpy.uint16, numpy.int32, numpy.uint32) shape_data_descriptors = ( ((6, ), DataAndMetadata.DataDescriptor(False, 0, 1)), # spectrum ((6, 4), DataAndMetadata.DataDescriptor(False, 1, 1)), # 1d collection of spectra ((6, 8, 10), DataAndMetadata.DataDescriptor(False, 2, 1)), # 2d collection of spectra ((6, 4), DataAndMetadata.DataDescriptor(True, 0, 1)), # sequence of spectra ((6, 4), DataAndMetadata.DataDescriptor(False, 0, 2)), # image ((6, 4, 2), DataAndMetadata.DataDescriptor(False, 1, 2)), # 1d collection of images ((6, 5, 4, 2), DataAndMetadata.DataDescriptor( False, 2, 2)), # 2d collection of images. not possible? ((6, 8, 10), DataAndMetadata.DataDescriptor(True, 0, 2)), # sequence of images ) for version in (3, 4): for array_type in array_types: for dtype in dtypes: for shape, data_descriptor_in in shape_data_descriptors: for signal_type in ( (None, "eels") if data_descriptor_in.datum_dimension_count == 1 else (None, )): # print(f"--- {array_type} {dtype} {shape} {data_descriptor_in} {signal_type}") s = io.BytesIO() with array_type(shape, dtype) as a: data_in = a.data dimensional_calibrations_in = list() for index, dimension in enumerate(shape): dimensional_calibrations_in.append( Calibration.Calibration( 1.0 + 0.1 * index, 2.0 + 0.2 * index, "µ" + "n" * index)) intensity_calibration_in = Calibration.Calibration( 4, 5, "six") metadata_in = dict() if signal_type: metadata_in.setdefault( "hardware_source", dict())["signal_type"] = signal_type xdata_in = DataAndMetadata.new_data_and_metadata( data_in, data_descriptor=data_descriptor_in, dimensional_calibrations= dimensional_calibrations_in, intensity_calibration= intensity_calibration_in, metadata=metadata_in) dm3_image_utils.save_image( xdata_in, s, version) s.seek(0) xdata = dm3_image_utils.load_image(s) self.assertTrue( numpy.array_equal(data_in, xdata.data)) self.assertEqual(data_descriptor_in, xdata.data_descriptor) self.assertEqual( dimensional_calibrations_in, xdata.dimensional_calibrations) self.assertEqual(intensity_calibration_in, xdata.intensity_calibration)
def disabled_test_specific_file(self): file_path = "/path/to/test.dm3" xdata = dm3_image_utils.load_image(file_path)