def read_epilogue(self):
"""Read the epilogue metadata."""
with utils.generic_open(self.filename, mode="rb") as fp_:
fp_.seek(self.mda['total_header_length'])
data = np.frombuffer(fp_.read(hrit_epilogue.itemsize),
dtype=hrit_epilogue,
count=1)
self.epilogue.update(recarray2dict(data))
def _read_file_or_file_like(self, shape, dtype):
# filename is likely to be a file-like object, already in memory
with utils.generic_open(self.filename, mode="rb") as fp:
no_elements = np.prod(shape)
fp.seek(self.mda['total_header_length'])
return np.frombuffer(
fp.read(np.dtype(dtype).itemsize * no_elements),
dtype=np.dtype(dtype),
count=no_elements.item()
).reshape(shape)
def test_generic_open_binary(self):
"""Test the bz2 file unzipping context manager using dummy binary data."""
dummy_binary_data = b'Hello'
dummy_binary_filename = 'dummy.dat'
with open(dummy_binary_filename, 'wb') as f:
f.write(dummy_binary_data)
with hf.generic_open(dummy_binary_filename, 'rb') as f:
read_binary_data = f.read()
assert read_binary_data == dummy_binary_data
dummy_binary_filename = 'dummy.dat.bz2'
with hf.bz2.open(dummy_binary_filename, 'wb') as f:
f.write(dummy_binary_data)
with hf.generic_open(dummy_binary_filename, 'rb') as f:
read_binary_data = f.read()
assert read_binary_data == dummy_binary_data
def test_generic_open_text(self):
"""Test the bz2 file unzipping context manager using dummy text data."""
dummy_text_data = 'Hello'
dummy_text_filename = 'dummy.txt'
with open(dummy_text_filename, 'w') as f:
f.write(dummy_text_data)
with hf.generic_open(dummy_text_filename, 'r') as f:
read_text_data = f.read()
assert read_text_data == dummy_text_data
dummy_text_filename = 'dummy.txt.bz2'
with hf.bz2.open(dummy_text_filename, 'wt') as f:
f.write(dummy_text_data)
with hf.generic_open(dummy_text_filename, 'rt') as f:
read_text_data = f.read()
assert read_text_data == dummy_text_data
def test_generic_open_FSFile_MemoryFileSystem(self):
"""Test the generic_open method with FSFile in MemoryFileSystem."""
mem_fs = MemoryFileSystem()
mem_file = MemoryFile(fs=mem_fs,
path="{}test.DAT".format(mem_fs.root_marker),
data=b"TEST")
mem_file.commit()
fsf = FSFile(mem_file)
with hf.generic_open(fsf) as file_object:
data = file_object.read()
assert data == b'TEST'
def test_generic_open_filename(self, open_mock):
"""Test the generic_open method with filename (str)."""
mock_fn_open = mock.MagicMock()
mock_fn_open.read.return_value = b'TEST'
open_mock.return_value = mock_fn_open
filename = "test.DAT"
with hf.generic_open(filename) as file_object:
data = file_object.read()
assert data == b'TEST'
assert mock_fn_open.read.called
def test_generic_open_BZ2File(self, bz2_mock):
"""Test the generic_open method with bz2 filename input."""
mock_bz2_open = mock.MagicMock()
mock_bz2_open.read.return_value = b'TEST'
bz2_mock.return_value = mock_bz2_open
filename = 'tester.DAT.bz2'
with hf.generic_open(filename) as file_object:
data = file_object.read()
assert data == b'TEST'
assert mock_bz2_open.read.called
def test_generic_open(self, bz2_mock):
"""Test the bz2 file unzipping context manager."""
mock_bz2_open = mock.MagicMock()
mock_bz2_open.read.return_value = b'TEST'
bz2_mock.return_value = mock_bz2_open
filename = 'tester.DAT.bz2'
with hf.generic_open(filename) as file_object:
data = file_object.read()
assert data == b'TEST'
assert mock_bz2_open.read.called
def read_prologue(self):
"""Read the prologue metadata."""
with utils.generic_open(self.filename, mode="rb") as fp_:
fp_.seek(self.mda['total_header_length'])
data = np.frombuffer(fp_.read(hrit_prologue.itemsize),
dtype=hrit_prologue,
count=1)
self.prologue.update(recarray2dict(data))
try:
impf = np.frombuffer(fp_.read(impf_configuration.itemsize),
dtype=impf_configuration,
count=1)[0]
except ValueError:
logger.info('No IMPF configuration field found in prologue.')
else:
self.prologue.update(recarray2dict(impf))
def _get_hd(self, hdr_info):
"""Open the file, read and get the basic file header info and set the mda dictionary."""
hdr_map, variable_length_headers, text_headers = hdr_info
with utils.generic_open(self.filename) as fp:
total_header_length = 16
while fp.tell() < total_header_length:
hdr_id = get_header_id(fp)
the_type = hdr_map[hdr_id['hdr_id']]
if the_type in variable_length_headers:
field_length = int(
(hdr_id['record_length'] - 3) / the_type.itemsize)
current_hdr = get_header_content(fp, the_type,
field_length)
key = variable_length_headers[the_type]
if key in self.mda:
if not isinstance(self.mda[key], list):
self.mda[key] = [self.mda[key]]
self.mda[key].append(current_hdr)
else:
self.mda[key] = current_hdr
elif the_type in text_headers:
field_length = int(
(hdr_id['record_length'] - 3) / the_type.itemsize)
char = list(the_type.fields.values())[0][0].char
new_type = np.dtype(char + str(field_length))
current_hdr = get_header_content(fp, new_type)[0]
self.mda[text_headers[the_type]] = current_hdr
else:
current_hdr = get_header_content(fp, the_type)[0]
self.mda.update(
dict(zip(current_hdr.dtype.names, current_hdr)))
total_header_length = self.mda['total_header_length']
self.mda.setdefault('number_of_bits_per_pixel', 10)
self.mda['projection_parameters'] = {
'a': 6378169.00,
'b': 6356583.80,
'h': 35785831.00,
# FIXME: find a reasonable SSP
'SSP_longitude': 0.0
}
self.mda['orbital_parameters'] = {}