def read_epilogue(self): """Read the epilogue metadata.""" with utils.generic_open(self.filename, mode="rb") as fp_: fp_.seek(self.mda['total_header_length']) data = np.frombuffer(fp_.read(hrit_epilogue.itemsize), dtype=hrit_epilogue, count=1) self.epilogue.update(recarray2dict(data))
def _read_file_or_file_like(self, shape, dtype): # filename is likely to be a file-like object, already in memory with utils.generic_open(self.filename, mode="rb") as fp: no_elements = np.prod(shape) fp.seek(self.mda['total_header_length']) return np.frombuffer( fp.read(np.dtype(dtype).itemsize * no_elements), dtype=np.dtype(dtype), count=no_elements.item() ).reshape(shape)
def test_generic_open_binary(self): """Test the bz2 file unzipping context manager using dummy binary data.""" dummy_binary_data = b'Hello' dummy_binary_filename = 'dummy.dat' with open(dummy_binary_filename, 'wb') as f: f.write(dummy_binary_data) with hf.generic_open(dummy_binary_filename, 'rb') as f: read_binary_data = f.read() assert read_binary_data == dummy_binary_data dummy_binary_filename = 'dummy.dat.bz2' with hf.bz2.open(dummy_binary_filename, 'wb') as f: f.write(dummy_binary_data) with hf.generic_open(dummy_binary_filename, 'rb') as f: read_binary_data = f.read() assert read_binary_data == dummy_binary_data
def test_generic_open_text(self): """Test the bz2 file unzipping context manager using dummy text data.""" dummy_text_data = 'Hello' dummy_text_filename = 'dummy.txt' with open(dummy_text_filename, 'w') as f: f.write(dummy_text_data) with hf.generic_open(dummy_text_filename, 'r') as f: read_text_data = f.read() assert read_text_data == dummy_text_data dummy_text_filename = 'dummy.txt.bz2' with hf.bz2.open(dummy_text_filename, 'wt') as f: f.write(dummy_text_data) with hf.generic_open(dummy_text_filename, 'rt') as f: read_text_data = f.read() assert read_text_data == dummy_text_data
def test_generic_open_FSFile_MemoryFileSystem(self): """Test the generic_open method with FSFile in MemoryFileSystem.""" mem_fs = MemoryFileSystem() mem_file = MemoryFile(fs=mem_fs, path="{}test.DAT".format(mem_fs.root_marker), data=b"TEST") mem_file.commit() fsf = FSFile(mem_file) with hf.generic_open(fsf) as file_object: data = file_object.read() assert data == b'TEST'
def test_generic_open_filename(self, open_mock): """Test the generic_open method with filename (str).""" mock_fn_open = mock.MagicMock() mock_fn_open.read.return_value = b'TEST' open_mock.return_value = mock_fn_open filename = "test.DAT" with hf.generic_open(filename) as file_object: data = file_object.read() assert data == b'TEST' assert mock_fn_open.read.called
def test_generic_open_BZ2File(self, bz2_mock): """Test the generic_open method with bz2 filename input.""" mock_bz2_open = mock.MagicMock() mock_bz2_open.read.return_value = b'TEST' bz2_mock.return_value = mock_bz2_open filename = 'tester.DAT.bz2' with hf.generic_open(filename) as file_object: data = file_object.read() assert data == b'TEST' assert mock_bz2_open.read.called
def test_generic_open(self, bz2_mock): """Test the bz2 file unzipping context manager.""" mock_bz2_open = mock.MagicMock() mock_bz2_open.read.return_value = b'TEST' bz2_mock.return_value = mock_bz2_open filename = 'tester.DAT.bz2' with hf.generic_open(filename) as file_object: data = file_object.read() assert data == b'TEST' assert mock_bz2_open.read.called
def read_prologue(self): """Read the prologue metadata.""" with utils.generic_open(self.filename, mode="rb") as fp_: fp_.seek(self.mda['total_header_length']) data = np.frombuffer(fp_.read(hrit_prologue.itemsize), dtype=hrit_prologue, count=1) self.prologue.update(recarray2dict(data)) try: impf = np.frombuffer(fp_.read(impf_configuration.itemsize), dtype=impf_configuration, count=1)[0] except ValueError: logger.info('No IMPF configuration field found in prologue.') else: self.prologue.update(recarray2dict(impf))
def _get_hd(self, hdr_info): """Open the file, read and get the basic file header info and set the mda dictionary.""" hdr_map, variable_length_headers, text_headers = hdr_info with utils.generic_open(self.filename) as fp: total_header_length = 16 while fp.tell() < total_header_length: hdr_id = get_header_id(fp) the_type = hdr_map[hdr_id['hdr_id']] if the_type in variable_length_headers: field_length = int( (hdr_id['record_length'] - 3) / the_type.itemsize) current_hdr = get_header_content(fp, the_type, field_length) key = variable_length_headers[the_type] if key in self.mda: if not isinstance(self.mda[key], list): self.mda[key] = [self.mda[key]] self.mda[key].append(current_hdr) else: self.mda[key] = current_hdr elif the_type in text_headers: field_length = int( (hdr_id['record_length'] - 3) / the_type.itemsize) char = list(the_type.fields.values())[0][0].char new_type = np.dtype(char + str(field_length)) current_hdr = get_header_content(fp, new_type)[0] self.mda[text_headers[the_type]] = current_hdr else: current_hdr = get_header_content(fp, the_type)[0] self.mda.update( dict(zip(current_hdr.dtype.names, current_hdr))) total_header_length = self.mda['total_header_length'] self.mda.setdefault('number_of_bits_per_pixel', 10) self.mda['projection_parameters'] = { 'a': 6378169.00, 'b': 6356583.80, 'h': 35785831.00, # FIXME: find a reasonable SSP 'SSP_longitude': 0.0 } self.mda['orbital_parameters'] = {}