def test_DmapRead_DmapWrite_stream_dmap(self):
"""
Test DmapRead and DmapWrite to read in a stream and write to a stream
to be read in again and compared.
"""
with bz2.open(rawacf_stream) as fp:
dmap_stream = fp.read()
dmap = pydarn.DmapRead(dmap_stream, True)
stream_data = dmap.read_records()
dmap_write = pydarn.DmapWrite()
dmap_write_stream = dmap_write.write_dmap_stream(stream_data)
dmap_read = pydarn.DmapRead(dmap_write_stream, True)
dmap_read_data = dmap_read.read_records()
self.dmap_compare(stream_data, dmap_read_data)
def test_integrity_check_corrupt_file2(self):
"""
Test test_initial_data_integrity on a corrupt file
Expected bahaviour: raises pydmap expection
"""
dmap = pydarn.DmapRead(corrupt_file2)
with self.assertRaises(pydarn.dmap_exceptions.NegativeByteError):
dmap.test_initial_data_integrity()
def test_read_corrupt_file1(self):
"""
Test read_records on a corrupt file
Expected bahaviour: raises pydmap expection
"""
dmap = pydarn.DmapRead(corrupt_file1)
with self.assertRaises(pydarn.dmap_exceptions.DmapDataTypeError):
dmap.read_records()
def test_DmapRead_stream_DmapWrite_file(self):
"""
Test DmapRead a stream in and DmapWrite to a file. DmapRead is
used to read the file to compare values.
"""
with bz2.open(rawacf_stream) as fp:
dmap_stream = fp.read()
dmap = pydarn.DmapRead(dmap_stream, True)
stream_data = dmap.read_records()
dmap_stream_data = dmap.get_dmap_records
dmap_write = pydarn.DmapWrite(stream_data)
dmap_write.write_dmap("test_rawacf.rawacf")
self.assertTrue(os.path.isfile("test_rawacf.rawacf"))
dmap = pydarn.DmapRead("test_rawacf.rawacf")
_ = dmap.read_records()
dmap_read_data = dmap.get_dmap_records
self.dmap_compare(dmap_stream_data, dmap_read_data)
def test_read_currupt_file2(self):
"""
Test read_records on a corrupt file
Expected bahaviour: raises pydmap expection
"""
dmap = pydarn.DmapRead(corrupt_file2)
with self.assertRaises(pydarn.dmap_exceptions.NegativeByteError):
dmap.read_records()
def test_DmapRead_dmap2dict_dict2dmap(self):
"""
Test DmapRead from a file and convert to a dictionary.
"""
dmap_read = pydarn.DmapRead(rawacf_file)
records = dmap_read.read_records()
records = dmap_read.get_dmap_records
dict_records = pydarn.dmap2dict(records)
records_2 = pydarn.dict2dmap(dict_records)
self.dmap_compare(records, records_2)
def test_DmapWrite_stream_DmapRead_dmap(self):
"""
Test DmapWrite write data to a stream, DmapRead read the stream.
"""
rawacf_data = copy.deepcopy(rawacf_data_sets.rawacf_data)
rawacf_write = pydarn.DmapWrite()
rawacf_stream = rawacf_write.write_dmap_stream(rawacf_data)
rawacf_read = pydarn.DmapRead(rawacf_stream, True)
rawacf_read_data = rawacf_read.read_records()
self.dmap_compare(rawacf_read_data, rawacf_data)
def test_DmapWrite_DmapRead_int8_scalar(self):
"""
Test integration DmapWrite and DmapRead to write char array.
"""
scalar = pydarn.DmapScalar('channel', 4, 1, 'c')
dmap_write = pydarn.DmapWrite([{'channel': scalar}])
dmap_scalar_bytes = dmap_write.dmap_scalar_to_bytes(scalar)
dmap = pydarn.DmapRead(dmap_scalar_bytes, True)
dmap_scalar = dmap.read_scalar()
self.assertEqual(scalar, dmap_scalar)
def test_integrity_check_dmap_file(self):
"""
Tests DmapRead test_initial_data_integrity
It should be able to read through the bytearray quickly
ensureing no curruption has occured in the file.
Behaviours: raising no exceptions
"""
file_path = rawacf_file
dm = pydarn.DmapRead(file_path)
dm.test_initial_data_integrity()
def test_DmapWrite_DmapRead_dmap_file(self):
"""
Test integration between DmapWrite and DmapRead from
a rawacf data written as file then read in and compared against.
"""
rawacf_data = copy.deepcopy(rawacf_data_sets.rawacf_data)
rawacf_write = pydarn.DmapWrite(rawacf_data, "test_rawacf.rawacf")
rawacf_write.write_dmap()
rawacf_read = pydarn.DmapRead("test_rawacf.rawacf")
rawacf_read_data = rawacf_read.read_records()
self.dmap_compare(rawacf_read_data, rawacf_data)
def test_DmapRead_DmapWrite_dmap_file(self):
"""
Test integration between DmapRead reading a file and DmapWrite
writing the file then reading it again to ensure the data didn't
change.
"""
dmap = pydarn.DmapRead(rawacf_file)
dmap_data = dmap.read_records()
writer = pydarn.DmapWrite(dmap_data, "test_rawacf.rawacf")
writer.write_dmap()
self.assertTrue(os.path.isfile("test_rawacf.rawacf"))
os.remove("test_rawacf.rawacf")
dmap_write = pydarn.DmapWrite(dmap_data)
dmap_write.write_dmap("test_rawacf.rawacf")
dmap_read = pydarn.DmapRead("test_rawacf.rawacf")
_ = dmap_read.read_records()
_ = dmap_read.get_dmap_records
os.remove("test_rawacf.rawacf")
def test_DmapRead_SDarnWrite_rawacf(self):
"""
Test DmapRead reading an rawacf and SDarnWrite writing
the rawacf file
"""
dmap = pydarn.DmapRead(rawacf_file)
dmap_data = dmap.read_records()
dmap_write = pydarn.SDarnWrite(dmap_data)
dmap_write.write_rawacf("test_rawacf.rawacf")
self.assertTrue(os.path.isfile("test_rawacf.rawacf"))
os.remove("test_rawacf.rawacf")
def test_write_read_int8_array(self):
"""
Test integrates DmapWrite and DmapRead to write and read an
int8 data type which is the char type for DMAP format.
"""
array = pydarn.DmapArray('channel', np.array([1, 0, 1], dtype=np.int8),
1, 'c', 1, [3])
dmap_write = pydarn.DmapWrite([{'channel': array}])
dmap_array_bytes = dmap_write.dmap_array_to_bytes(array)
dmap = pydarn.DmapRead(dmap_array_bytes, True)
dmap_array = dmap.read_array(len(dmap_array_bytes))
self.compare_dmap_array(array, dmap_array)
def test_DmapRead_SDarnWrite_SDarnRead_fitacf(self):
"""
Test DmapRead reading a fitacf file then writing it with SDarnWrite
then reading it again with SDarnRead
"""
dmap = pydarn.DmapRead(fitacf_file)
dmap_data = dmap.read_records()
dmap_write = pydarn.SDarnWrite(dmap_data)
dmap_write.write_fitacf("test_fitacf.fitacf")
darn_read = pydarn.SDarnRead("test_fitacf.fitacf")
fitacf_data = darn_read.read_fitacf()
self.dmap_compare(dmap_data, fitacf_data)
os.remove("test_fitacf.fitacf")
def test_DmapWrite_DmapRead_scalar(self):
"""
This test integrates DmapWrite and DmapRead for
writing a single scalar and then reading it in.
Behaviour: No change to the scalar being written then read in.
"""
scalar = pydarn.DmapScalar('stid', 5, 3, 'i')
dmap_write = pydarn.DmapWrite([{'stid': scalar}])
dmap_scalar_bytes = dmap_write.dmap_scalar_to_bytes(scalar)
dmap = pydarn.DmapRead(dmap_scalar_bytes, True)
dmap = dmap.read_scalar()
self.assertEqual(scalar, dmap)
def test_open_dmap_file(self):
"""
Tests DmapRead's constructor on opening a rawacf.
It should be able to open the file, read it and convert to bytearray.
Checks:
- bytearray instance is created from reading in the file
- bytearray is not empty
"""
file_path = fitacf_file
dm = pydarn.DmapRead(file_path)
self.assertIsInstance(dm.dmap_bytearr, bytearray)
self.assertGreater(dm.dmap_end_bytes, 0)
def test_DmapRead_SDarnWrite_SDarnRead_map(self):
"""
Test DmapRead reading a map file then writing it with SDarnWrite
then reading it again with SDarnRead
"""
dmap = pydarn.DmapRead(map_file)
dmap_data = dmap.read_records()
dmap_write = pydarn.SDarnWrite(dmap_data)
dmap_write.write_map("test_map.map")
darn_read = pydarn.SDarnRead("test_map.map")
map_data = darn_read.read_map()
self.dmap_compare(dmap_data, map_data)
os.remove("test_map.map")
def test_DmapRead_SDarnWrite_SDarnRead_grid(self):
"""
Test DmapRead reading a grid file then writing it with SDarnWrite
then reading it again with SDarnRead
"""
dmap = pydarn.DmapRead(grid_file)
dmap_data = dmap.read_records()
dmap_write = pydarn.SDarnWrite(dmap_data)
dmap_write.write_grid("test_grid.grid")
darn_read = pydarn.SDarnRead("test_grid.grid")
grid_data = darn_read.read_grid()
self.dmap_compare(dmap_data, grid_data)
os.remove("test_grid.grid")
def test_DmapRead_SDarnWrite_SDarnRead_iqdat(self):
"""
Test SDarnRead reads from a stream and SDarnWrite writes
to a iqdat file
"""
dmap = pydarn.DmapRead(iqdat_file)
dmap_data = dmap.read_records()
dmap_write = pydarn.SDarnWrite(dmap_data)
dmap_write.write_iqdat("test_iqdat.iqdat")
darn_read = pydarn.SDarnRead("test_iqdat.iqdat")
iqdat_data = darn_read.read_iqdat()
self.dmap_compare(dmap_data, iqdat_data)
os.remove("test_iqdat.iqdat")
def test_dict2dmap_SDarnWrite_rawacf(self):
"""
Test dict2dmap to convert a dictionary to dmap then SDarnWrite write
rawacf file
"""
rawacf_dict_data = copy.deepcopy(rawacf_dict_sets.rawacf_dict_data)
dmap_rawacf = pydarn.dict2dmap(rawacf_dict_data)
darn_read = pydarn.SDarnWrite(dmap_rawacf)
darn_read.write_rawacf("test_rawacf.rawacf")
dmap_read = pydarn.DmapRead("test_rawacf.rawacf")
dmap_data = dmap_read.read_records()
dmap_data = dmap_read.get_dmap_records
self.dmap_compare(dmap_data, dmap_rawacf)
os.remove("test_rawacf.rawacf")
def test_DmapRead_stream_SDarnWrite_file_map(self):
"""
Test DmapRead to read in a stream then have SDarnWrite the
stream to file
"""
with bz2.open(map_stream) as fp:
dmap_stream = fp.read()
dmap = pydarn.DmapRead(dmap_stream, True)
dmap_stream_data = dmap.read_records()
dmap_write = pydarn.SDarnWrite(dmap_stream_data)
dmap_write.write_map("test_map.map")
dmap = pydarn.SDarnRead("test_map.map")
dmap_data = dmap.read_map()
self.dmap_compare(dmap_stream_data, dmap_data)
os.remove("test_map.map")
def test_DmapWrite_DmapRead_array(self):
"""
This test integrates DmapWrite and DmapRead for
writing and reading an array.
Behaviour: No change to the array
"""
array = pydarn.DmapArray('xcf',
np.array([4.3, 3.5, 2.3], dtype=np.float32),
4, 'f', 1, [3])
dmap_write = pydarn.DmapWrite([{'xcf': array}])
dmap_array_bytes = dmap_write.dmap_array_to_bytes(array)
dmap = pydarn.DmapRead(dmap_array_bytes, True)
dmap_array = dmap.read_array(len(dmap_array_bytes))
self.compare_dmap_array(array, dmap_array)
def test_DmapRead_stream_SDarnWrite_file_fitacf(self):
"""
Test DmapRead to read in a stream then have SDarnWrite the
stream to file
"""
with bz2.open(fitacf_stream) as fp:
dmap_stream = fp.read()
dmap = pydarn.DmapRead(dmap_stream, True)
stream_data = dmap.read_records()
dmap_stream_data = dmap.get_dmap_records
dmap_write = pydarn.SDarnWrite(stream_data)
dmap_write.write_fitacf("test_fitacf.fitacf")
dmap = pydarn.SDarnRead("test_fitacf.fitacf")
dmap_data = dmap.read_fitacf()
dmap_data = dmap.get_dmap_records
self.dmap_compare(dmap_stream_data, dmap_data)
os.remove("test_fitacf.fitacf")
def test_read_dmap_file(self):
"""
Tests DmapRead test read_dmap.
Behaviour: raising no exceptions
"""
file_path = fitacf_file
dm = pydarn.DmapRead(file_path)
dmap_records = dm.read_records()
dmap_records = dm.get_dmap_records
self.assertIsInstance(dmap_records, collections.deque)
self.assertIsInstance(dmap_records[0], collections.OrderedDict)
self.assertIsInstance(dmap_records[4]['bmnum'], pydarn.DmapScalar)
self.assertIsInstance(dmap_records[1]['ptab'], pydarn.DmapArray)
self.assertIsInstance(dmap_records[7]['channel'].value, int)
self.assertIsInstance(dmap_records[2]['ltab'].value, np.ndarray)
self.assertEqual(dmap_records[0]['ptab'].dimension, 1)
self.assertEqual(dmap_records[50]['gflg'].value[1], 0)
def test_dmap_read_corrupt_stream(self):
"""
Test read_records on a corrupt stream. Read in compressed
file which returns a byte object, then insert some random
bytes to produce a corrupt stream.
Expected bahaviour: raises pydmap expection
"""
with bz2.open(rawacf_stream) as fp:
dmap_stream = fp.read()
# need to convert to byte array for mutability
# since bytes are immutable.
corrupt_stream = bytearray(dmap_stream[0:36])
corrupt_stream[36:40] = bytearray(str(os.urandom(4)).encode('utf-8'))
corrupt_stream[40:] = dmap_stream[37:]
dmap = pydarn.DmapRead(corrupt_stream, True)
with self.assertRaises(pydarn.dmap_exceptions.DmapDataError):
dmap.read_records()
def test_DmapWrite_DmapRead_dict2dmap_dict2dmap(self):
"""
Test Convert dictionary to dmap from dict2dmap then
write with DmapWrite to be read in with DmapRead and
converted back to a dictionary with dmap2dict.
"""
dmap_dict = [{'stid': 56,
'channel': 0,
'software': 'RST',
'xcf': np.array([2.5, 3.456, 34.56, -4.5],
dtype=np.float32),
'gflg': np.array([1, 0, 4, 2], np.int8)}]
dmap_records = pydarn.dict2dmap(dmap_dict)
dmap_write = pydarn.DmapWrite(dmap_records, 'test_dmap.dmap')
dmap_write.write_dmap()
dmap_read = pydarn.DmapRead("test_dmap.dmap")
records = dmap_read.read_records()
self.dmap_compare(dmap_dict, records)
dmap_dict2 = pydarn.dict2dmap(records)
self.dmap_compare(dmap_records, dmap_dict2)
def test_dict2dmap_DmapWrite_DmapRead_stream_dmap2dict(self):
"""
Test convert dict to dmap with dict2dmap then DmapWrite to write
to a stream to be read in by
DmapRead and converted back to a dictionary
with dict2dmap
"""
dmap_dict = [{'stid': 56,
'channel': 0,
'software': 'RST',
'xcf': np.array([2.5, 3.456, 34.56, -4.5],
dtype=np.float32),
'gflg': np.array([1, 0, 4, 2], np.int8)}]
dmap_records = pydarn.dict2dmap(dmap_dict)
dmap_write = pydarn.DmapWrite(dmap_records)
dmap_stream = dmap_write.write_dmap_stream()
dmap_read = pydarn.DmapRead(dmap_stream, True)
records = dmap_read.read_records()
self.dmap_compare(dmap_dict, records)
dmap_dict2 = pydarn.dict2dmap(records)
self.dmap_compare(dmap_records, dmap_dict2)
def test_dmap_read_stream(self):
"""
Test read_records on dmap data stream.
The dmap data stream is formed from compressed
bzip2 file that returns a bytes object.
Checks:
- returns correct data structures
- returns excpected values
"""
# bz2 opens the compressed file into a data
# stream of bytes without actually uncompressing the file
with bz2.open(rawacf_stream) as fp:
dmap_stream = fp.read()
dmap = pydarn.DmapRead(dmap_stream, True)
dmap_data = dmap.read_records()
dmap_data = dmap.get_dmap_records
self.assertIsInstance(dmap_data, collections.deque)
self.assertIsInstance(dmap_data[0], collections.OrderedDict)
self.assertIsInstance(dmap_data[4]['channel'], pydarn.DmapScalar)
self.assertIsInstance(dmap_data[1]['ptab'], pydarn.DmapArray)
self.assertIsInstance(dmap_data[7]['channel'].value, int)
self.assertIsInstance(dmap_data[2]['xcfd'].value, np.ndarray)
self.assertEqual(dmap_data[0]['xcfd'].dimension, 3)
def test_DmapWrite_DmapRead_dmap2dict(self):
"""
Test DmapWrite writing a dmap data set to be read in by DmapRead
and convert to a dictionary by dmap2dict and compared.
"""
dmap_dict = [{'RST.version': '4.1',
'stid': 3,
'FAC.vel': np.array([2.5, 3.5, 4.0], dtype=np.float32)},
{'RST.version': '4.1',
'stid': 3,
'FAC.vel': np.array([1, 0, 1], dtype=np.int8)},
{'RST.version': '4.1',
'stid': 3,
'FAC.vel': np.array([5.7, 2.34, -0.2],
dtype=np.float32)}]
dmap_data = copy.deepcopy(dmap_data_sets.dmap_data)
dmap_write = pydarn.DmapWrite(dmap_data)
dmap_stream = dmap_write.write_dmap_stream()
dmap_read = pydarn.DmapRead(dmap_stream, True)
dmap_records = dmap_read.read_records()
dmap_records = dmap_read.get_dmap_records
self.dmap_compare(dmap_records, dmap_data)
dmap_dict2 = pydarn.dmap2dict(dmap_records)
self.dict_compare(dmap_dict, dmap_dict2)
def dmap_mem_profile():
fitacf_file = "../testfiles/20180220.0001.00.rkn.3.0.fitacf"
dmap = pydarn.DmapRead(fitacf_file)
dmap_data = dmap.read_records()
return dmap_data
