def test_save_load_cycle(): sig_reload = None signal = hs.load(file2) serial = signal.original_metadata['blockfile_header']['Acquisition_time'] date, time, timezone = serial_date_to_ISO_format(serial) nt.assert_equal(signal.metadata.General.original_filename, 'test2.blo') nt.assert_equal(signal.metadata.General.date, date) nt.assert_equal(signal.metadata.General.time, time) nt.assert_equal(signal.metadata.General.time_zone, timezone) nt.assert_equal( signal.metadata.General.notes, "Precession angle : \r\nPrecession Frequency : \r\nCamera gamma : on") signal.save(save_path, overwrite=True) sig_reload = hs.load(save_path) np.testing.assert_equal(signal.data, sig_reload.data) nt.assert_equal(signal.axes_manager.as_dictionary(), sig_reload.axes_manager.as_dictionary()) nt.assert_equal(signal.original_metadata.as_dictionary(), sig_reload.original_metadata.as_dictionary()) # change original_filename to make the metadata of both signals equals sig_reload.metadata.General.original_filename = signal.metadata.General.original_filename assert_deep_almost_equal(signal.metadata.as_dictionary(), sig_reload.metadata.as_dictionary()) nt.assert_equal(signal.metadata.General.date, sig_reload.metadata.General.date) nt.assert_equal(signal.metadata.General.time, sig_reload.metadata.General.time) nt.assert_is_instance(signal, hs.signals.Signal2D) # Delete reference to close memmap file! del sig_reload gc.collect() _remove_file(save_path)
def test_save_load_cycle(save_path): sig_reload = None signal = hs.load(FILE2) serial = signal.original_metadata['blockfile_header']['Acquisition_time'] date, time, timezone = serial_date_to_ISO_format(serial) assert signal.metadata.General.original_filename == 'test2.blo' assert signal.metadata.General.date == date assert signal.metadata.General.time == time assert signal.metadata.General.time_zone == timezone assert ( signal.metadata.General.notes == "Precession angle : \r\nPrecession Frequency : \r\nCamera gamma : on") signal.save(save_path, overwrite=True) sig_reload = hs.load(save_path) np.testing.assert_equal(signal.data, sig_reload.data) assert (signal.axes_manager.as_dictionary() == sig_reload.axes_manager.as_dictionary()) assert (signal.original_metadata.as_dictionary() == sig_reload.original_metadata.as_dictionary()) # change original_filename to make the metadata of both signals equals sig_reload.metadata.General.original_filename = signal.metadata.General.original_filename assert_deep_almost_equal(signal.metadata.as_dictionary(), sig_reload.metadata.as_dictionary()) assert ( signal.metadata.General.date == sig_reload.metadata.General.date) assert ( signal.metadata.General.time == sig_reload.metadata.General.time) assert isinstance(signal, hs.signals.Signal2D) # Delete reference to close memmap file! del sig_reload
def test_save_load_cycle(save_path, convert_units): sig_reload = None signal = hs.load(FILE2, convert_units=convert_units) serial = signal.original_metadata['blockfile_header']['Acquisition_time'] date, time, timezone = serial_date_to_ISO_format(serial) assert signal.metadata.General.original_filename == 'test2.blo' assert signal.metadata.General.date == date assert signal.metadata.General.time == time assert signal.metadata.General.time_zone == timezone assert ( signal.metadata.General.notes == "Precession angle : \r\nPrecession Frequency : \r\nCamera gamma : on") signal.save(save_path, overwrite=True) sig_reload = hs.load(save_path, convert_units=convert_units) np.testing.assert_equal(signal.data, sig_reload.data) assert (signal.axes_manager.as_dictionary() == sig_reload.axes_manager.as_dictionary()) assert (signal.original_metadata.as_dictionary() == sig_reload.original_metadata.as_dictionary()) # change original_filename to make the metadata of both signals equals sig_reload.metadata.General.original_filename = signal.metadata.General.original_filename assert_deep_almost_equal(signal.metadata.as_dictionary(), sig_reload.metadata.as_dictionary()) assert (signal.metadata.General.date == sig_reload.metadata.General.date) assert (signal.metadata.General.time == sig_reload.metadata.General.time) assert isinstance(signal, hs.signals.Signal2D) # Delete reference to close memmap file! del sig_reload
def test_serial_date_to_ISO_format(): iso_1 = dtt.serial_date_to_ISO_format(serial1) dt1_local = dt1.astimezone(tz.tzlocal()) assert iso_1[0] == dt1_local.date().isoformat() assert iso_1[1] == dt1_local.time().isoformat() assert iso_1[2] == dt1_local.tzname() iso_2 = dtt.serial_date_to_ISO_format(serial2) dt2_local = dt2.astimezone(tz.tzlocal()) assert iso_2[0] == dt2_local.date().isoformat() # The below line will/can fail due to accuracy loss when converting to serial date: # We therefore truncate milli/micro seconds assert iso_2[1][:8] == dt2_local.time().isoformat() assert iso_2[2] == dt2_local.tzname() iso_3 = dtt.serial_date_to_ISO_format(serial3) dt3_aware = dt3.replace(tzinfo=tz.tzutc()) dt3_local = dt3_aware.astimezone(tz.tzlocal()) assert iso_3[0] == dt3_local.date().isoformat() assert iso_3[1] == dt3_local.time().isoformat() assert iso_3[2] == dt3_local.tzname()
def test_save_load_cycle(save_path, convert_units): sig_reload = None signal = hs.load(FILE2, convert_units=convert_units) serial = signal.original_metadata["blockfile_header"]["Acquisition_time"] date, time, timezone = serial_date_to_ISO_format(serial) assert signal.metadata.General.original_filename == "test2.blo" assert signal.metadata.General.date == date assert signal.metadata.General.time == time assert signal.metadata.General.time_zone == timezone assert ( signal.metadata.General.notes == "Precession angle : \r\nPrecession Frequency : \r\nCamera gamma : on") signal.save(save_path, overwrite=True) sig_reload = hs.load(save_path, convert_units=convert_units) np.testing.assert_equal(signal.data, sig_reload.data) assert (signal.axes_manager.as_dictionary() == sig_reload.axes_manager.as_dictionary()) assert (signal.original_metadata.as_dictionary() == sig_reload.original_metadata.as_dictionary()) # change original_filename to make the metadata of both signals equals sig_reload.metadata.General.original_filename = ( signal.metadata.General.original_filename) # assert file reading tests here, then delete so we can compare # entire metadata structure at once: plugin = 'hyperspy.io_plugins.blockfile' assert signal.metadata.General.FileIO.Number_0.operation == 'load' assert signal.metadata.General.FileIO.Number_0.io_plugin == plugin assert signal.metadata.General.FileIO.Number_1.operation == 'save' assert signal.metadata.General.FileIO.Number_1.io_plugin == plugin assert sig_reload.metadata.General.FileIO.Number_0.operation == 'load' assert sig_reload.metadata.General.FileIO.Number_0.io_plugin == plugin del signal.metadata.General.FileIO del sig_reload.metadata.General.FileIO assert_deep_almost_equal(signal.metadata.as_dictionary(), sig_reload.metadata.as_dictionary()) assert signal.metadata.General.date == sig_reload.metadata.General.date assert signal.metadata.General.time == sig_reload.metadata.General.time assert isinstance(signal, hs.signals.Signal2D) # Delete reference to close memmap file! del sig_reload
def file_reader(filename, endianess='<', mmap_mode='c', lazy=False, **kwds): _logger.debug("Reading blockfile: %s" % filename) metadata = {} # Makes sure we open in right mode: if '+' in mmap_mode or ('write' in mmap_mode and 'copyonwrite' != mmap_mode): if lazy: raise ValueError("Lazy loading does not support in-place writing") f = open(filename, 'r+b') else: f = open(filename, 'rb') _logger.debug("File opened") # Get header header = np.fromfile(f, dtype=get_header_dtype_list(endianess), count=1) if header['MAGIC'][0] not in magics: warnings.warn("Blockfile has unrecognized header signature. " "Will attempt to read, but correcteness not guaranteed!") header = sarray2dict(header) note = f.read(header['Data_offset_1'] - f.tell()) note = note.strip(b'\x00') header['Note'] = note.decode() _logger.debug("File header: " + str(header)) NX, NY = int(header['NX']), int(header['NY']) DP_SZ = int(header['DP_SZ']) if header['SDP']: SDP = 100. / header['SDP'] else: SDP = Undefined original_metadata = {'blockfile_header': header} # Get data: # A Virtual BF/DF is stored first # offset1 = header['Data_offset_1'] # f.seek(offset1) # data_pre = np.array(f.read(NX*NY), dtype=endianess+'u1' # ).squeeze().reshape((NX, NY), order='C').T # Then comes actual blockfile offset2 = header['Data_offset_2'] if not lazy: f.seek(offset2) data = np.fromfile(f, dtype=endianess + 'u1') else: data = np.memmap(f, mode=mmap_mode, offset=offset2, dtype=endianess + 'u1') try: data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) except ValueError: warnings.warn( 'Blockfile header dimensions larger than file size! ' 'Will attempt to load by zero padding incomplete frames.') # Data is stored DP by DP: pw = [(0, NX * NY * (DP_SZ * DP_SZ + 6) - data.size)] data = np.pad(data, pw, mode='constant') data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) # Every frame is preceeded by a 6 byte sequence (AA 55, and then a 4 byte # integer specifying frame number) data = data[:, :, 6:] data = data.reshape((NY, NX, DP_SZ, DP_SZ), order='C').squeeze() units = ['nm', 'nm', 'cm', 'cm'] names = ['y', 'x', 'dy', 'dx'] scales = [header['SY'], header['SX'], SDP, SDP] date, time, time_zone = serial_date_to_ISO_format(header['Acquisition_time']) metadata = {'General': {'original_filename': os.path.split(filename)[1], 'date': date, 'time': time, 'time_zone': time_zone, 'notes': header['Note']}, "Signal": {'signal_type': "diffraction", 'record_by': 'image', }, } # Create the axis objects for each axis dim = data.ndim axes = [ { 'size': data.shape[i], 'index_in_array': i, 'name': names[i], 'scale': scales[i], 'offset': 0.0, 'units': units[i], } for i in range(dim)] dictionary = {'data': data, 'axes': axes, 'metadata': metadata, 'original_metadata': original_metadata, 'mapping': mapping, } f.close() return [dictionary, ]
def file_reader(filename, endianess='<', mmap_mode=None, lazy=False, **kwds): _logger.debug("Reading blockfile: %s" % filename) metadata = {} if mmap_mode is None: mmap_mode = 'r' if lazy else 'c' # Makes sure we open in right mode: if '+' in mmap_mode or ('write' in mmap_mode and 'copyonwrite' != mmap_mode): if lazy: raise ValueError("Lazy loading does not support in-place writing") f = open(filename, 'r+b') else: f = open(filename, 'rb') _logger.debug("File opened") # Get header header = np.fromfile(f, dtype=get_header_dtype_list(endianess), count=1) if header['MAGIC'][0] not in magics: warnings.warn("Blockfile has unrecognized header signature. " "Will attempt to read, but correcteness not guaranteed!") header = sarray2dict(header) note = f.read(header['Data_offset_1'] - f.tell()) # It seems it uses "\x00" for padding, so we remove it try: header['Note'] = note.decode("latin1").strip("\x00") except BaseException: # Not sure about the encoding so, if it fails, we carry on _logger.warning( "Reading the Note metadata of this file failed. " "You can help improving " "HyperSpy by reporting the issue in " "https://github.com/hyperspy/hyperspy") _logger.debug("File header: " + str(header)) NX, NY = int(header['NX']), int(header['NY']) DP_SZ = int(header['DP_SZ']) if header['SDP']: SDP = 100. / header['SDP'] else: SDP = Undefined original_metadata = {'blockfile_header': header} # Get data: # A Virtual BF/DF is stored first # offset1 = header['Data_offset_1'] # f.seek(offset1) # data_pre = np.array(f.read(NX*NY), dtype=endianess+'u1' # ).squeeze().reshape((NX, NY), order='C').T # Then comes actual blockfile offset2 = header['Data_offset_2'] if not lazy: f.seek(offset2) data = np.fromfile(f, dtype=endianess + 'u1') else: data = np.memmap(f, mode=mmap_mode, offset=offset2, dtype=endianess + 'u1') try: data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) except ValueError: warnings.warn( 'Blockfile header dimensions larger than file size! ' 'Will attempt to load by zero padding incomplete frames.') # Data is stored DP by DP: pw = [(0, NX * NY * (DP_SZ * DP_SZ + 6) - data.size)] data = np.pad(data, pw, mode='constant') data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) # Every frame is preceeded by a 6 byte sequence (AA 55, and then a 4 byte # integer specifying frame number) data = data[:, :, 6:] data = data.reshape((NY, NX, DP_SZ, DP_SZ), order='C').squeeze() units = ['nm', 'nm', 'cm', 'cm'] names = ['y', 'x', 'dy', 'dx'] scales = [header['SY'], header['SX'], SDP, SDP] date, time, time_zone = serial_date_to_ISO_format( header['Acquisition_time']) metadata = {'General': {'original_filename': os.path.split(filename)[1], 'date': date, 'time': time, 'time_zone': time_zone, 'notes': header['Note']}, "Signal": {'signal_type': "diffraction", 'record_by': 'image', }, } # Create the axis objects for each axis dim = data.ndim axes = [ { 'size': data.shape[i], 'index_in_array': i, 'name': names[i], 'scale': scales[i], 'offset': 0.0, 'units': units[i], } for i in range(dim)] dictionary = {'data': data, 'axes': axes, 'metadata': metadata, 'original_metadata': original_metadata, 'mapping': mapping, } f.close() return [dictionary, ]
def file_reader(filename, endianess="<", mmap_mode=None, lazy=False, **kwds): _logger.debug("Reading blockfile: %s" % filename) metadata = {} if mmap_mode is None: mmap_mode = "r" if lazy else "c" # Makes sure we open in right mode: if "+" in mmap_mode or ("write" in mmap_mode and "copyonwrite" != mmap_mode): if lazy: raise ValueError("Lazy loading does not support in-place writing") f = open(filename, "r+b") else: f = open(filename, "rb") _logger.debug("File opened") # Get header header = np.fromfile(f, dtype=get_header_dtype_list(endianess), count=1) if header["MAGIC"][0] not in magics: warnings.warn( "Blockfile has unrecognized header signature. " "Will attempt to read, but correcteness not guaranteed!", UserWarning, ) header = sarray2dict(header) note = f.read(header["Data_offset_1"] - f.tell()) # It seems it uses "\x00" for padding, so we remove it try: header["Note"] = note.decode("latin1").strip("\x00") except BaseException: # Not sure about the encoding so, if it fails, we carry on _logger.warning("Reading the Note metadata of this file failed. " "You can help improving " "HyperSpy by reporting the issue in " "https://github.com/hyperspy/hyperspy") _logger.debug("File header: " + str(header)) NX, NY = int(header["NX"]), int(header["NY"]) DP_SZ = int(header["DP_SZ"]) if header["SDP"]: SDP = 100.0 / header["SDP"] else: SDP = Undefined original_metadata = {"blockfile_header": header} # Get data: # TODO A Virtual BF/DF is stored first, may be loaded as navigator in future # offset1 = header['Data_offset_1'] # f.seek(offset1) # navigator = np.fromfile(f, dtype=endianess+"u1", shape=(NX, NY)).T # Then comes actual blockfile offset2 = header["Data_offset_2"] if not lazy: f.seek(offset2) data = np.fromfile(f, dtype=endianess + "u1") else: data = np.memmap(f, mode=mmap_mode, offset=offset2, dtype=endianess + "u1") try: data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) except ValueError: warnings.warn( "Blockfile header dimensions larger than file size! " "Will attempt to load by zero padding incomplete frames.") # Data is stored DP by DP: pw = [(0, NX * NY * (DP_SZ * DP_SZ + 6) - data.size)] data = np.pad(data, pw, mode="constant") data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) # Every frame is preceeded by a 6 byte sequence (AA 55, and then a 4 byte # integer specifying frame number) data = data[:, :, 6:] data = data.reshape((NY, NX, DP_SZ, DP_SZ), order="C").squeeze() units = ["nm", "nm", "cm", "cm"] names = ["y", "x", "dy", "dx"] scales = [header["SY"], header["SX"], SDP, SDP] date, time, time_zone = serial_date_to_ISO_format( header["Acquisition_time"]) metadata = { "General": { "original_filename": os.path.split(filename)[1], "date": date, "time": time, "time_zone": time_zone, "notes": header["Note"], }, "Signal": { "signal_type": "diffraction", "record_by": "image", }, } # Create the axis objects for each axis dim = data.ndim axes = [{ "size": data.shape[i], "index_in_array": i, "name": names[i], "scale": scales[i], "offset": 0.0, "units": units[i], } for i in range(dim)] dictionary = { "data": data, "axes": axes, "metadata": metadata, "original_metadata": original_metadata, "mapping": mapping, } f.close() return [ dictionary, ]