def setUp(self): self.rawacf_array_data = copy.deepcopy(borealis_array_rawacf_data) self.bfiq_array_data = copy.deepcopy(borealis_array_bfiq_data) # write some v0.4 data self.bfiq_test_file = "test_bfiq.bfiq.hdf5" _ = pydarnio.BorealisWrite(self.bfiq_test_file, self.bfiq_array_data, 'bfiq', 'array') self.rawacf_test_file = "test_rawacf.rawacf.hdf5" _ = pydarnio.BorealisWrite(self.rawacf_test_file, self.rawacf_array_data, 'rawacf', 'array') # get v0.5 data from file self.bfiqv05_test_file = borealis_site_bfiq_file_v05 self.rawacfv05_test_file = borealis_site_rawacf_file_v05
def test_read_array_write_site_antennas_iq(self): """ Test reading, restructuring, writing, restructuring antennas_iq. Checks: - arrays that pass the read from a file can then be written as records - arrays restructured, written as records, read as records, restructured back to arrays are the same as original arrays """ dm = pydarnio.BorealisRead(self.source_antennas_iq_array_file, 'antennas_iq', 'array') arrays = dm.arrays writer = pydarnio.BorealisWrite(self.write_antennas_iq_site_file, dm.records, 'antennas_iq', 'site') del dm, writer gc.collect() self.assertTrue(os.path.isfile(self.write_antennas_iq_site_file)) dm2 = pydarnio.BorealisRead(self.write_antennas_iq_site_file, 'antennas_iq', 'site') new_arrays = dm2.arrays # restructuring happens here dictionaries_are_same = self.check_dictionaries_are_same( arrays, new_arrays) self.assertTrue(dictionaries_are_same) os.remove(self.write_antennas_iq_site_file) del dm2, arrays, new_arrays
def test_read_site_write_array_bfiq(self): """ Test reading, restructuring, writing, restructuring bfiq. Checks: - records that pass the read from a file can then be written as arrays - records restructured, written as arrays, read as arrays, restructured back to records are the same as original records """ dm = pydarnio.BorealisRead(self.source_bfiq_site_file, 'bfiq', 'site') records = dm.records arrays = dm.arrays # restructuring happens here _ = pydarnio.BorealisWrite(self.write_bfiq_array_file, arrays, 'bfiq', 'array') del dm, arrays self.assertTrue(os.path.isfile(self.write_bfiq_array_file)) dm2 = pydarnio.BorealisRead(self.write_bfiq_array_file, 'bfiq', 'array') new_records = dm2.records # restructuring happens here dictionaries_are_same = self.check_dictionaries_are_same( records, new_records) self.assertTrue(dictionaries_are_same) os.remove(self.write_bfiq_array_file) del _, dm2, records, new_records
def setUp(self): self.rawacf_array_data = copy.deepcopy(borealis_array_rawacf_data) self.bfiq_array_data = copy.deepcopy(borealis_array_bfiq_data) self.bfiq_file = 'test_bfiq.bfiq.hdf5' self.rawacf_file = 'test_rawacf.rawacf.hdf5' self.iqdat_array_darn_file = './test_iqdat_array_file.dmap' self.rawacf_array_darn_file = './test_rawacf_array_file.dmap' _ = pydarnio.BorealisWrite(self.bfiq_file, self.bfiq_array_data, 'bfiq', 'array') bfiq_reader = pydarnio.BorealisRead(self.bfiq_file, 'bfiq', 'array') self.bfiq_site_data = bfiq_reader.records self.iqdat_site_darn_file = './test_iqdat_site_file.dmap' _ = pydarnio.BorealisWrite(self.rawacf_file, self.rawacf_array_data, 'rawacf', 'array') rawacf_reader = pydarnio.BorealisRead(self.rawacf_file, 'rawacf', 'array') self.rawacf_site_data = rawacf_reader.records self.rawacf_site_darn_file = './test_rawacf_site_file.dmap'
def test_extra_field_array_rawacf(self): """ Tests write_rawacf method - writes a rawacf structure file for the given data Expected behaviour ------------------ Raises BorealisExtraFieldError because the rawacf data has an extra field dummy """ self.rawacf_array_extra_field['dummy'] = 'dummy' try: _ = pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5", self.rawacf_array_extra_field, 'rawacf', 'array') except pydarnio.borealis_exceptions.BorealisExtraFieldError as err: self.assertEqual(err.fields, {'dummy'})
def test_writing_array_rawacf(self): """ Tests write_rawacf method - writes a rawacf file Expected behaviour ------------------ Rawacf file is produced """ test_file = "test_rawacf.rawacf.hdf5" _ = pydarnio.BorealisWrite(test_file, self.rawacf_array_data, 'rawacf', 'array') self.assertTrue(os.path.isfile(test_file)) reader = pydarnio.BorealisRead(test_file, 'rawacf', 'array') data = reader.arrays dictionaries_are_same =\ self.check_dictionaries_are_same(data, self.rawacf_array_data) self.assertTrue(dictionaries_are_same) os.remove(test_file)
def test_missing_field_array_bfiq(self): """ Tests write_bfiq method - writes a bfiq structure file for the given data Expected behaviour ------------------ Raises BorealisFieldMissingError - because the bfiq data is missing field antenna_arrays_order """ del self.bfiq_array_missing_field['antenna_arrays_order'] try: _ = pydarnio.BorealisWrite("test_bfiq.bfiq.hdf5", self.bfiq_array_missing_field, 'bfiq', 'array') except pydarnio.borealis_exceptions.BorealisFieldMissingError as err: self.assertEqual(err.fields, {'antenna_arrays_order'})
def test_incorrect_data_format_array_bfiq(self): """ Tests write_bfiq method - writes a bfiq structure file for the given data Expected Behaviour ------------------- Raises BorealisDataFormatTypeError because the bfiq data has the wrong type for the first_range_rtt field """ self.bfiq_array_incorrect_fmt['first_range_rtt'] = 5 try: _ = pydarnio.BorealisWrite("test_bfiq.bfiq.hdf5", self.bfiq_array_incorrect_fmt, 'bfiq', 'array') except pydarnio.borealis_exceptions.BorealisDataFormatTypeError as err: self.assertEqual(err.incorrect_types['first_range_rtt'], "<class 'numpy.float32'>")
def test_extra_field_site_rawacf(self): """ Tests write_rawacf method - writes a rawacf structure file for the given data Expected behaviour ------------------ Raises BorealisExtraFieldError because the rawacf data has an extra field dummy """ keys = sorted(list(self.rawacf_site_extra_field.keys())) self.rawacf_site_extra_field[keys[0]]['dummy'] = 'dummy' try: pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5", self.rawacf_site_extra_field, 'rawacf', 'site') except pydarnio.borealis_exceptions.BorealisExtraFieldError as err: self.assertEqual(err.fields, {'dummy'}) self.assertEqual(err.record_name, keys[0])
def test_missing_field_array_rawacf(self): """ Tests write_rawacf method - writes a rawacf structure file for the given data Expected behaviour ------------------ Raises BorealisFieldMissingError - because the rawacf data is missing field num_sequences """ del self.rawacf_array_missing_field['num_sequences'] try: _ = pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5", self.rawacf_array_missing_field, 'rawacf', 'array') except pydarnio.borealis_exceptions.BorealisFieldMissingError as err: self.assertEqual(err.fields, {'num_sequences'})
def test_missing_field_site_bfiq(self): """ Tests write_bfiq method - writes a bfiq structure file for the given data Expected behaviour ------------------ Raises BorealisFieldMissingError - because the bfiq data is missing field antenna_arrays_order """ keys = sorted(list(self.bfiq_site_missing_field.keys())) del self.bfiq_site_missing_field[keys[0]]['antenna_arrays_order'] try: _ = pydarnio.BorealisWrite("test_bfiq.bfiq.hdf5", self.bfiq_site_missing_field, 'bfiq', 'site') except pydarnio.borealis_exceptions.BorealisFieldMissingError as err: self.assertEqual(err.fields, {'antenna_arrays_order'}) self.assertEqual(err.record_name, keys[0])
def test_read_write_site_bfiq(self): """ Test reading and then writing site bfiq data to a file. Checks: - records that pass the read from a file can then be written - records written and then read are the same as original """ dm = pydarnio.BorealisRead(self.source_bfiq_site_file, 'bfiq', 'site') records = dm.records _ = pydarnio.BorealisWrite(self.write_bfiq_site_file, records, 'bfiq', 'site') self.assertTrue(os.path.isfile(self.write_bfiq_site_file)) dm2 = pydarnio.BorealisRead(self.write_bfiq_site_file, 'bfiq', 'site') new_records = dm2.records dictionaries_are_same = self.check_dictionaries_are_same( records, new_records) self.assertTrue(dictionaries_are_same) os.remove(self.write_bfiq_site_file) del _, dm, dm2, records, new_records
def test_writing_site_bfiq(self): """ Tests write_bfiq method - writes a bfiq file Expected behaviour ------------------ bfiq file is produced """ test_file = "./test_bfiq.bfiq.hdf5" pydarnio.BorealisWrite(test_file, self.bfiq_site_data, 'bfiq', 'site') # only testing the file is created since it should only be created # at the last step after all checks have passed # Testing the integrity of the insides of the file will be part of # integration testing since we need BorealisSiteRead for that. self.assertTrue(os.path.isfile(test_file)) reader = pydarnio.BorealisRead(test_file, 'bfiq', 'site') records = reader.records dictionaries_are_same = \ self.check_dictionaries_are_same(records, self.bfiq_site_data) self.assertTrue(dictionaries_are_same) os.remove(test_file)
def test_incorrect_data_format_site_rawacf(self): """ Tests write_rawacf method - writes a rawacf structure file for the given data Expected Behaviour ------------------- Raises BorealisDataFormatTypeError because the rawacf data has the wrong type for the scan_start_marker field """ keys = sorted(list(self.rawacf_site_incorrect_fmt.keys())) self.rawacf_site_incorrect_fmt[keys[0]]['scan_start_marker'] = 1 try: pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5", self.rawacf_site_incorrect_fmt, 'rawacf', 'site') except pydarnio.borealis_exceptions.BorealisDataFormatTypeError as err: self.assertEqual(err.incorrect_types['scan_start_marker'], "<class 'numpy.bool_'>") self.assertEqual(err.record_name, keys[0])
def test_missing_field_site_rawacf(self): """ Tests write_rawacf method - writes a rawacf structure file for the given data Expected behaviour ------------------ Raises BorealisFieldMissingError - because the rawacf data is missing field num_sequences """ keys = sorted(list(self.rawacf_site_missing_field.keys())) del self.rawacf_site_missing_field[keys[0]]['num_sequences'] try: pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5", self.rawacf_site_missing_field, 'rawacf', 'site') except pydarnio.borealis_exceptions.BorealisFieldMissingError as err: self.assertEqual(err.fields, {'num_sequences'}) self.assertEqual(err.record_name, keys[0])
def test_incorrect_data_format_array_rawacf(self): """ Tests write_rawacf method - writes a rawacf structure file for the given data Expected Behaviour ------------------- Raises BorealisDataFormatTypeError because the rawacf data has the wrong type for the scan_start_marker field """ num_records =\ self.rawacf_array_incorrect_fmt['scan_start_marker'].shape[0] self.rawacf_array_incorrect_fmt['scan_start_marker'] = \ np.array([1] * num_records) try: _ = pydarnio.BorealisWrite("test_rawacf.rawacf.hdf5", self.rawacf_array_incorrect_fmt, 'rawacf', 'array') except pydarnio.borealis_exceptions.BorealisDataFormatTypeError as err: self.assertEqual(err.incorrect_types['scan_start_marker'], "np.ndarray of <class 'numpy.bool_'>")
def test_rawacf2darnrawacf(self): """ Test BorealisConvert by providing both site data and array data. Test that the dmap records are the same for both (site data and array data are from the same source, should produce same dmap output.) This tests that the restructuring is working before the dmap conversion """ # self.rawacf_file already written in setUp array_converter =\ pydarnio.BorealisConvert(self.rawacf_file, "rawacf", self.rawacf_array_darn_file, 0, borealis_file_structure='array') self.assertTrue(os.path.isfile(self.rawacf_array_darn_file)) darn_reader = pydarnio.SDarnRead(self.rawacf_array_darn_file) rawacf_array_records = darn_reader.read_rawacf() os.remove(self.rawacf_array_darn_file) os.remove(self.rawacf_file) _ = pydarnio.BorealisWrite(self.rawacf_file, self.rawacf_site_data, 'rawacf', 'site') site_converter =\ pydarnio.BorealisConvert(self.rawacf_file, "rawacf", self.rawacf_site_darn_file, 0, borealis_file_structure='site') self.assertTrue(os.path.isfile(self.rawacf_site_darn_file)) darn_reader = pydarnio.SDarnRead(self.rawacf_site_darn_file) rawacf_site_records = darn_reader.read_rawacf() os.remove(self.rawacf_site_darn_file) os.remove(self.rawacf_file) for record_num, record in enumerate(array_converter.sdarn_dict): dictionaries_are_same =\ self.check_dictionaries_are_same(record, site_converter. sdarn_dict[record_num]) self.assertTrue(dictionaries_are_same) del (array_converter, site_converter, darn_reader, rawacf_site_records, rawacf_array_records)
def test_read_write_array_bfiq(self): """ Test reading and then writing array structured bfiq data to a file. Checks: - arrays that pass the read from a file can then be written - arrays written and then read are the same as original """ dm = pydarnio.BorealisRead(self.source_bfiq_array_file, 'bfiq', 'array') arrays = dm.arrays _ = pydarnio.BorealisWrite(self.write_bfiq_array_file, arrays, 'bfiq', 'array') self.assertTrue(os.path.isfile(self.write_bfiq_array_file)) dm2 = pydarnio.BorealisRead(self.write_bfiq_array_file, 'bfiq', 'array') new_arrays = dm2.arrays dictionaries_are_same = self.check_dictionaries_are_same( arrays, new_arrays) self.assertTrue(dictionaries_are_same) os.remove(self.write_bfiq_array_file) del _, dm, dm2, arrays, new_arrays
def restructure(infile_name, outfile_name, infile_type, infile_structure, outfile_structure): """ This method restructures filename of structure "file_structure" into "final_structure". Parameters ---------- infile_name: str Name of the original file. outfile_name: str Name of the restructured file. infile_type: str Borealis file type of the files. infile_structure: str The current write structure of the file. One of 'array' or 'site'. outfile_structure: str The desired write structure of the file. One of 'array', 'site', 'iqdat', or 'dmap'. """ # dmap and iqdat are not borealis formats, so they are handled specially if outfile_structure == 'dmap' or outfile_structure == 'iqdat': pydarnio.BorealisConvert(infile_name, infile_type, outfile_name, borealis_file_structure=infile_structure) return # Get data from the file reader = pydarnio.BorealisRead(infile_name, infile_type, infile_structure) # Get data in correct format for writing to output file if outfile_structure == 'site': data = reader.records else: data = reader.arrays # Write to output file pydarnio.BorealisWrite(outfile_name, data, infile_type, outfile_structure)