def setUp(self):
self.rawacf_array_data = copy.deepcopy(borealis_array_rawacf_data)
self.bfiq_array_data = copy.deepcopy(borealis_array_bfiq_data)
# write some v0.4 data
self.bfiq_test_file = "test_bfiq.bfiq.hdf5"
_ = pydarnio.BorealisWrite(self.bfiq_test_file, self.bfiq_array_data,
'bfiq', 'array')
self.rawacf_test_file = "test_rawacf.rawacf.hdf5"
_ = pydarnio.BorealisWrite(self.rawacf_test_file,
self.rawacf_array_data, 'rawacf', 'array')
# get v0.5 data from file
self.bfiqv05_test_file = borealis_site_bfiq_file_v05
self.rawacfv05_test_file = borealis_site_rawacf_file_v05
def test_read_array_write_site_antennas_iq(self):
"""
Test reading, restructuring, writing, restructuring antennas_iq.
Checks:
- arrays that pass the read from a file can then be written
as records
- arrays restructured, written as records, read as records,
restructured back to arrays are the same as original arrays
"""
dm = pydarnio.BorealisRead(self.source_antennas_iq_array_file,
'antennas_iq', 'array')
arrays = dm.arrays
writer = pydarnio.BorealisWrite(self.write_antennas_iq_site_file,
dm.records, 'antennas_iq', 'site')
del dm, writer
gc.collect()
self.assertTrue(os.path.isfile(self.write_antennas_iq_site_file))
dm2 = pydarnio.BorealisRead(self.write_antennas_iq_site_file,
'antennas_iq', 'site')
new_arrays = dm2.arrays # restructuring happens here
dictionaries_are_same = self.check_dictionaries_are_same(
arrays, new_arrays)
self.assertTrue(dictionaries_are_same)
os.remove(self.write_antennas_iq_site_file)
del dm2, arrays, new_arrays
def test_read_site_write_array_bfiq(self):
"""
Test reading, restructuring, writing, restructuring bfiq.
Checks:
- records that pass the read from a file can then be written
as arrays
- records restructured, written as arrays, read as arrays,
restructured back to records are the same as original records
"""
dm = pydarnio.BorealisRead(self.source_bfiq_site_file, 'bfiq', 'site')
records = dm.records
arrays = dm.arrays # restructuring happens here
_ = pydarnio.BorealisWrite(self.write_bfiq_array_file, arrays, 'bfiq',
'array')
del dm, arrays
self.assertTrue(os.path.isfile(self.write_bfiq_array_file))
dm2 = pydarnio.BorealisRead(self.write_bfiq_array_file, 'bfiq',
'array')
new_records = dm2.records # restructuring happens here
dictionaries_are_same = self.check_dictionaries_are_same(
records, new_records)
self.assertTrue(dictionaries_are_same)
os.remove(self.write_bfiq_array_file)
del _, dm2, records, new_records
def setUp(self):
self.rawacf_array_data = copy.deepcopy(borealis_array_rawacf_data)
self.bfiq_array_data = copy.deepcopy(borealis_array_bfiq_data)
self.bfiq_file = 'test_bfiq.bfiq.hdf5'
self.rawacf_file = 'test_rawacf.rawacf.hdf5'
self.iqdat_array_darn_file = './test_iqdat_array_file.dmap'
self.rawacf_array_darn_file = './test_rawacf_array_file.dmap'
_ = pydarnio.BorealisWrite(self.bfiq_file, self.bfiq_array_data,
'bfiq', 'array')
bfiq_reader = pydarnio.BorealisRead(self.bfiq_file, 'bfiq', 'array')
self.bfiq_site_data = bfiq_reader.records
self.iqdat_site_darn_file = './test_iqdat_site_file.dmap'
_ = pydarnio.BorealisWrite(self.rawacf_file, self.rawacf_array_data,
'rawacf', 'array')
rawacf_reader = pydarnio.BorealisRead(self.rawacf_file, 'rawacf',
'array')
self.rawacf_site_data = rawacf_reader.records
self.rawacf_site_darn_file = './test_rawacf_site_file.dmap'
def test_extra_field_array_rawacf(self):
"""
Tests write_rawacf method - writes a rawacf structure file for the
given data
Expected behaviour
------------------
Raises BorealisExtraFieldError because the rawacf data
has an extra field dummy
"""
self.rawacf_array_extra_field['dummy'] = 'dummy'
try:
_ = pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5",
self.rawacf_array_extra_field, 'rawacf',
'array')
except pydarnio.borealis_exceptions.BorealisExtraFieldError as err:
self.assertEqual(err.fields, {'dummy'})
def test_writing_array_rawacf(self):
"""
Tests write_rawacf method - writes a rawacf file
Expected behaviour
------------------
Rawacf file is produced
"""
test_file = "test_rawacf.rawacf.hdf5"
_ = pydarnio.BorealisWrite(test_file, self.rawacf_array_data, 'rawacf',
'array')
self.assertTrue(os.path.isfile(test_file))
reader = pydarnio.BorealisRead(test_file, 'rawacf', 'array')
data = reader.arrays
dictionaries_are_same =\
self.check_dictionaries_are_same(data, self.rawacf_array_data)
self.assertTrue(dictionaries_are_same)
os.remove(test_file)
def test_missing_field_array_bfiq(self):
"""
Tests write_bfiq method - writes a bfiq structure file for the
given data
Expected behaviour
------------------
Raises BorealisFieldMissingError - because the bfiq data is
missing field antenna_arrays_order
"""
del self.bfiq_array_missing_field['antenna_arrays_order']
try:
_ = pydarnio.BorealisWrite("test_bfiq.bfiq.hdf5",
self.bfiq_array_missing_field, 'bfiq',
'array')
except pydarnio.borealis_exceptions.BorealisFieldMissingError as err:
self.assertEqual(err.fields, {'antenna_arrays_order'})
def test_incorrect_data_format_array_bfiq(self):
"""
Tests write_bfiq method - writes a bfiq structure file for the
given data
Expected Behaviour
-------------------
Raises BorealisDataFormatTypeError because the bfiq data has the
wrong type for the first_range_rtt field
"""
self.bfiq_array_incorrect_fmt['first_range_rtt'] = 5
try:
_ = pydarnio.BorealisWrite("test_bfiq.bfiq.hdf5",
self.bfiq_array_incorrect_fmt, 'bfiq',
'array')
except pydarnio.borealis_exceptions.BorealisDataFormatTypeError as err:
self.assertEqual(err.incorrect_types['first_range_rtt'],
"<class 'numpy.float32'>")
def test_extra_field_site_rawacf(self):
"""
Tests write_rawacf method - writes a rawacf structure file for the
given data
Expected behaviour
------------------
Raises BorealisExtraFieldError because the rawacf data
has an extra field dummy
"""
keys = sorted(list(self.rawacf_site_extra_field.keys()))
self.rawacf_site_extra_field[keys[0]]['dummy'] = 'dummy'
try:
pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5",
self.rawacf_site_extra_field, 'rawacf',
'site')
except pydarnio.borealis_exceptions.BorealisExtraFieldError as err:
self.assertEqual(err.fields, {'dummy'})
self.assertEqual(err.record_name, keys[0])
def test_missing_field_array_rawacf(self):
"""
Tests write_rawacf method - writes a rawacf structure file for the
given data
Expected behaviour
------------------
Raises BorealisFieldMissingError - because the rawacf data is
missing field num_sequences
"""
del self.rawacf_array_missing_field['num_sequences']
try:
_ = pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5",
self.rawacf_array_missing_field,
'rawacf', 'array')
except pydarnio.borealis_exceptions.BorealisFieldMissingError as err:
self.assertEqual(err.fields, {'num_sequences'})
def test_missing_field_site_bfiq(self):
"""
Tests write_bfiq method - writes a bfiq structure file for the
given data
Expected behaviour
------------------
Raises BorealisFieldMissingError - because the bfiq data is
missing field antenna_arrays_order
"""
keys = sorted(list(self.bfiq_site_missing_field.keys()))
del self.bfiq_site_missing_field[keys[0]]['antenna_arrays_order']
try:
_ = pydarnio.BorealisWrite("test_bfiq.bfiq.hdf5",
self.bfiq_site_missing_field, 'bfiq',
'site')
except pydarnio.borealis_exceptions.BorealisFieldMissingError as err:
self.assertEqual(err.fields, {'antenna_arrays_order'})
self.assertEqual(err.record_name, keys[0])
def test_read_write_site_bfiq(self):
"""
Test reading and then writing site bfiq data to a file.
Checks:
- records that pass the read from a file can then be written
- records written and then read are the same as original
"""
dm = pydarnio.BorealisRead(self.source_bfiq_site_file, 'bfiq', 'site')
records = dm.records
_ = pydarnio.BorealisWrite(self.write_bfiq_site_file, records, 'bfiq',
'site')
self.assertTrue(os.path.isfile(self.write_bfiq_site_file))
dm2 = pydarnio.BorealisRead(self.write_bfiq_site_file, 'bfiq', 'site')
new_records = dm2.records
dictionaries_are_same = self.check_dictionaries_are_same(
records, new_records)
self.assertTrue(dictionaries_are_same)
os.remove(self.write_bfiq_site_file)
del _, dm, dm2, records, new_records
def test_writing_site_bfiq(self):
"""
Tests write_bfiq method - writes a bfiq file
Expected behaviour
------------------
bfiq file is produced
"""
test_file = "./test_bfiq.bfiq.hdf5"
pydarnio.BorealisWrite(test_file, self.bfiq_site_data, 'bfiq', 'site')
# only testing the file is created since it should only be created
# at the last step after all checks have passed
# Testing the integrity of the insides of the file will be part of
# integration testing since we need BorealisSiteRead for that.
self.assertTrue(os.path.isfile(test_file))
reader = pydarnio.BorealisRead(test_file, 'bfiq', 'site')
records = reader.records
dictionaries_are_same = \
self.check_dictionaries_are_same(records, self.bfiq_site_data)
self.assertTrue(dictionaries_are_same)
os.remove(test_file)
def test_incorrect_data_format_site_rawacf(self):
"""
Tests write_rawacf method - writes a rawacf structure file for the
given data
Expected Behaviour
-------------------
Raises BorealisDataFormatTypeError because the rawacf data has the
wrong type for the scan_start_marker field
"""
keys = sorted(list(self.rawacf_site_incorrect_fmt.keys()))
self.rawacf_site_incorrect_fmt[keys[0]]['scan_start_marker'] = 1
try:
pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5",
self.rawacf_site_incorrect_fmt, 'rawacf',
'site')
except pydarnio.borealis_exceptions.BorealisDataFormatTypeError as err:
self.assertEqual(err.incorrect_types['scan_start_marker'],
"<class 'numpy.bool_'>")
self.assertEqual(err.record_name, keys[0])
def test_missing_field_site_rawacf(self):
"""
Tests write_rawacf method - writes a rawacf structure file for the
given data
Expected behaviour
------------------
Raises BorealisFieldMissingError - because the rawacf data is
missing field num_sequences
"""
keys = sorted(list(self.rawacf_site_missing_field.keys()))
del self.rawacf_site_missing_field[keys[0]]['num_sequences']
try:
pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5",
self.rawacf_site_missing_field, 'rawacf',
'site')
except pydarnio.borealis_exceptions.BorealisFieldMissingError as err:
self.assertEqual(err.fields, {'num_sequences'})
self.assertEqual(err.record_name, keys[0])
def test_incorrect_data_format_array_rawacf(self):
"""
Tests write_rawacf method - writes a rawacf structure file for the
given data
Expected Behaviour
-------------------
Raises BorealisDataFormatTypeError because the rawacf data has the
wrong type for the scan_start_marker field
"""
num_records =\
self.rawacf_array_incorrect_fmt['scan_start_marker'].shape[0]
self.rawacf_array_incorrect_fmt['scan_start_marker'] = \
np.array([1] * num_records)
try:
_ = pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5",
self.rawacf_array_incorrect_fmt,
'rawacf', 'array')
except pydarnio.borealis_exceptions.BorealisDataFormatTypeError as err:
self.assertEqual(err.incorrect_types['scan_start_marker'],
"np.ndarray of <class 'numpy.bool_'>")
def test_rawacf2darnrawacf(self):
"""
Test BorealisConvert by providing both site data and array data.
Test that the dmap records are the same for both (site data and
array data are from the same source, should produce same dmap output.)
This tests that the restructuring is working before the dmap conversion
"""
# self.rawacf_file already written in setUp
array_converter =\
pydarnio.BorealisConvert(self.rawacf_file, "rawacf",
self.rawacf_array_darn_file, 0,
borealis_file_structure='array')
self.assertTrue(os.path.isfile(self.rawacf_array_darn_file))
darn_reader = pydarnio.SDarnRead(self.rawacf_array_darn_file)
rawacf_array_records = darn_reader.read_rawacf()
os.remove(self.rawacf_array_darn_file)
os.remove(self.rawacf_file)
_ = pydarnio.BorealisWrite(self.rawacf_file, self.rawacf_site_data,
'rawacf', 'site')
site_converter =\
pydarnio.BorealisConvert(self.rawacf_file, "rawacf",
self.rawacf_site_darn_file, 0,
borealis_file_structure='site')
self.assertTrue(os.path.isfile(self.rawacf_site_darn_file))
darn_reader = pydarnio.SDarnRead(self.rawacf_site_darn_file)
rawacf_site_records = darn_reader.read_rawacf()
os.remove(self.rawacf_site_darn_file)
os.remove(self.rawacf_file)
for record_num, record in enumerate(array_converter.sdarn_dict):
dictionaries_are_same =\
self.check_dictionaries_are_same(record,
site_converter.
sdarn_dict[record_num])
self.assertTrue(dictionaries_are_same)
del (array_converter, site_converter, darn_reader, rawacf_site_records,
rawacf_array_records)
def test_read_write_array_bfiq(self):
"""
Test reading and then writing array structured bfiq data to a file.
Checks:
- arrays that pass the read from a file can then be written
- arrays written and then read are the same as original
"""
dm = pydarnio.BorealisRead(self.source_bfiq_array_file, 'bfiq',
'array')
arrays = dm.arrays
_ = pydarnio.BorealisWrite(self.write_bfiq_array_file, arrays, 'bfiq',
'array')
self.assertTrue(os.path.isfile(self.write_bfiq_array_file))
dm2 = pydarnio.BorealisRead(self.write_bfiq_array_file, 'bfiq',
'array')
new_arrays = dm2.arrays
dictionaries_are_same = self.check_dictionaries_are_same(
arrays, new_arrays)
self.assertTrue(dictionaries_are_same)
os.remove(self.write_bfiq_array_file)
del _, dm, dm2, arrays, new_arrays
def restructure(infile_name, outfile_name, infile_type, infile_structure,
outfile_structure):
"""
This method restructures filename of structure "file_structure" into "final_structure".
Parameters
----------
infile_name: str
Name of the original file.
outfile_name: str
Name of the restructured file.
infile_type: str
Borealis file type of the files.
infile_structure: str
The current write structure of the file. One of 'array' or 'site'.
outfile_structure: str
The desired write structure of the file. One of 'array', 'site', 'iqdat', or 'dmap'.
"""
# dmap and iqdat are not borealis formats, so they are handled specially
if outfile_structure == 'dmap' or outfile_structure == 'iqdat':
pydarnio.BorealisConvert(infile_name,
infile_type,
outfile_name,
borealis_file_structure=infile_structure)
return
# Get data from the file
reader = pydarnio.BorealisRead(infile_name, infile_type, infile_structure)
# Get data in correct format for writing to output file
if outfile_structure == 'site':
data = reader.records
else:
data = reader.arrays
# Write to output file
pydarnio.BorealisWrite(outfile_name, data, infile_type, outfile_structure)
