def test_write_compressed(self, mocked_gzip):
mocked_gzip.return_value = ' '
with tempfile.TemporaryDirectory() as dirname:
path_with_extension = Path(dirname) / (DUMMY_FILE_NAME +
iisr_io.ARCHIVE_EXTENSION)
# When called without archive extension
path = Path(dirname) / DUMMY_FILE_NAME
with iisr_io.open_data_file(path, 'w',
compress_on_write=True) as writer:
self.assertIsInstance(writer, iisr_io.DataFileWriter)
self.assertTrue(mocked_gzip.called)
self.assertTrue(path_with_extension.exists())
mocked_gzip.called = False
# When called with archive extension
with iisr_io.open_data_file(path_with_extension,
'w',
compress_on_write=True) as writer:
self.assertIsInstance(writer, iisr_io.DataFileWriter)
self.assertTrue(mocked_gzip.called)
self.assertTrue(path_with_extension.exists())
def test_read(self):
with tempfile.TemporaryDirectory() as dirname:
path = Path(dirname) / DUMMY_FILE_NAME
with self.assertRaises(FileNotFoundError), iisr_io.open_data_file(
path, 'r'):
pass
path.touch()
with iisr_io.open_data_file(path, 'r') as reader:
self.assertIsInstance(reader, iisr_io.DataFileReader)
def test_read_compressed(self, mocked_gzip, mocked_tempfile):
with tempfile.TemporaryDirectory() as dirname:
path = Path(dirname) / (DUMMY_FILE_NAME +
iisr_io.ARCHIVE_EXTENSION)
with self.assertRaises(FileNotFoundError):
with iisr_io.open_data_file(path, 'r'):
pass
path.touch()
with iisr_io.open_data_file(path, 'r') as reader:
self.assertIsInstance(reader, iisr_io.DataFileReader)
self.assertTrue(mocked_gzip.called)
self.assertTrue(mocked_tempfile.called)
def test_read_real_file(self):
with iisr_io.open_data_file(TEST_REAL_FILEPATH) as data_file:
series_list = list(data_file)
self.assertEqual(len(series_list), 100)
first_series = series_list[0] # type: iisr_io.TimeSeries
self.assertEqual(first_series.time_mark.year, 2015)
self.assertEqual(first_series.time_mark.month, 6)
self.assertEqual(first_series.time_mark.day, 6)
unique_freqs = set()
unique_len = set()
unique_channels = set()
for series in series_list:
unique_freqs.add(series.parameters.frequency)
unique_len.add(series.parameters.pulse_length)
unique_channels.add(series.parameters.channel)
for freq in [155.5, 155.8, 159.5, 159.8]:
freq = Frequency(freq, 'MHz')
self.assertIn(freq, unique_freqs)
for len_ in [0, 200, 700, 900]:
len_ = TimeUnit(len_, 'us')
self.assertIn(len_, unique_len)
for ch in [0, 1, 2, 3]:
ch = Channel(ch)
self.assertIn(ch, unique_channels)
def test_read_with_selector(self):
valid_params = {
'frequencies': [Frequency(155.5, 'MHz')],
'channels': [Channel(0), Channel(2)],
'pulse_lengths': [TimeUnit(200, 'us')],
'pulse_types': ['long'],
}
filter_ = iisr_io.SeriesSelector(**valid_params)
test_parameters = []
for fr, ch, len_ in it.product([155.5, 159.5], [0, 2], [200, 700]):
test_parameters.append(
get_test_parameters(freq=fr, pulse_len=len_, channel=ch))
with tempfile.TemporaryDirectory() as dirname:
test_filepath = Path(dirname) / DUMMY_FILE_NAME
with iisr_io.open_data_file(test_filepath, 'w') as data_file:
for param in test_parameters:
n_samples = param.n_samples
test_quad_i = np.random.randint(-2**15 + 1, 2**15,
n_samples)
test_quad_q = np.random.randint(-2**15 + 1, 2**15,
n_samples)
test_quadratures = test_quad_i + 1j * test_quad_q
series = iisr_io.TimeSeries(DEFAULT_DATETIME, param,
test_quadratures)
package = iisr_io.TimeSeriesPackage(
DEFAULT_DATETIME, [series])
data_file.write(package)
# Check if selector correct
with iisr_io.open_data_file(test_filepath,
series_selector=filter_) as reader:
series_list = list(reader)
self.assertEqual(len(series_list), 2)
for series in series_list:
self.assertEqual(series.parameters.frequency,
valid_params['frequencies'][0])
self.assertIn(series.parameters.channel,
valid_params['channels'])
self.assertEqual(series.parameters.pulse_length,
valid_params['pulse_lengths'][0])
self.assertEqual(series.parameters.pulse_type,
valid_params['pulse_types'][0])
def test_write_read_reciprocity(self):
with tempfile.TemporaryDirectory() as dirname:
test_file_path = Path(dirname) / DUMMY_FILE_NAME
# Create test options
test_parameters = get_test_parameters()
time_mark = DEFAULT_DATETIME
n_samples = test_parameters.n_samples
test_quad_i = np.random.randint(-2**15 + 1, 2**15, n_samples)
test_quad_q = np.random.randint(-2**15 + 1, 2**15, n_samples)
test_quadratures = test_quad_i + 1j * test_quad_q
test_series = iisr_io.TimeSeries(time_mark, test_parameters,
test_quadratures)
test_package = iisr_io.TimeSeriesPackage(time_mark, [test_series])
with iisr_io.open_data_file(test_file_path, 'w') as writer:
writer.write(test_package)
with iisr_io.open_data_file(test_file_path, 'r') as reader:
series = next(reader.read_series())
# Time
self.assertEqual(time_mark, series.time_mark)
# Quadratures
self.assertEqual(len(test_quadratures), len(series.quadratures))
np.testing.assert_equal(test_quadratures, series.quadratures)
# Check test options against read options
self.assertEqual(test_parameters.n_samples,
series.parameters.n_samples)
self.assertEqual(test_parameters.frequency,
series.parameters.frequency)
self.assertEqual(test_parameters.pulse_type,
series.parameters.pulse_type)
self.assertEqual(test_parameters.pulse_length,
series.parameters.pulse_length)
self.assertEqual(test_parameters.sampling_frequency,
series.parameters.sampling_frequency)
self.assertEqual(test_parameters.channel, series.parameters.channel)
self.assertEqual(test_parameters.phase_code,
series.parameters.phase_code)
self.assertEqual(test_parameters.total_delay,
series.parameters.total_delay)
def test_read(self):
with make_random_test_file() as (test_file_path, test_series_list), \
iisr_io.open_data_file(test_file_path) as reader:
series = next(reader.read_series())
self.assertIsInstance(series.time_mark, datetime)
self.assertIsInstance(series.parameters, iisr_io.SeriesParameters)
self.assertEqual(series.time_mark, test_series_list[0].time_mark)
self.assertEqual(series.parameters, test_series_list[0].parameters)
self.assertIsInstance(series.quadratures, np.ndarray)
np.testing.assert_almost_equal(series.quadratures,
test_series_list[0].quadratures)
def _run_read(self, hours_shift, fix_time_bug):
with tempfile.TemporaryDirectory() as dirname:
test_file_path = Path(dirname) / DUMMY_FILE_NAME
# Create test options
n_samples = 64
test_parameters = [
get_test_parameters(channel=ch, n_samples=n_samples)
for ch in self.channels
]
minutes_shift = random.randint(-5, 5) # +-5 minutes maximum
time_marks = [
DEFAULT_DATETIME + timedelta(seconds=10 * i)
for i in range(self.n_marks)
]
for i in range(self.shift_start_idx, self.shift_stop_idx):
time_marks[i] += timedelta(hours=hours_shift,
minutes=minutes_shift)
test_quad_i = np.random.randint(-2**15 + 1, 2**15, n_samples)
test_quad_q = np.random.randint(-2**15 + 1, 2**15, n_samples)
test_quadratures = test_quad_i + 1j * test_quad_q
test_series = []
test_packages = []
for time_mark in time_marks:
tm_series = []
for params in test_parameters:
series = iisr_io.TimeSeries(time_mark, params,
test_quadratures)
test_series.append(series)
tm_series.append(series)
test_packages.append(
iisr_io.TimeSeriesPackage(time_mark, tm_series))
with iisr_io.open_data_file(test_file_path, 'w') as writer:
for package in test_packages:
writer.write(package)
with open(str(test_file_path), 'rb') as file:
reader = iisr_io.DataFileReader(file,
file_info=get_file_info(),
fix_time_lag_bug=fix_time_bug)
read_series = list(reader.read_series())
self.assertEqual(self.n_marks * len(self.channels),
len(read_series))
return test_series, read_series
def make_random_test_file(n_unique_series=2, n_time_marks=2, time_step_sec=1):
"""Create test_file with random series"""
def gen_unique_parameters():
freqs = [154., 155.5, 156.8, 158., 159.5]
random.shuffle(freqs)
channels = [0, 1, 2, 3]
random.shuffle(channels)
pulse_len = [200, 500, 700, 900]
random.shuffle(pulse_len)
phase_code = [0, 5]
random.shuffle(phase_code)
# Channels must change first, as in original data
for code, length, freq, ch in it.product(phase_code, pulse_len, freqs,
channels):
yield get_test_parameters(freq=freq,
channel=ch,
pulse_len=length,
phase_code=code)
generator = gen_unique_parameters()
parameter_sets = []
for i in range(n_unique_series):
parameter_sets.append(generator.__next__())
packages_list = []
all_series_list = []
for i in range(n_time_marks):
time_mark = DEFAULT_DATETIME + timedelta(seconds=time_step_sec * i)
series_list = []
for parameters in parameter_sets:
n_samples = parameters.n_samples
quad_i = np.random.randint(-2**15 + 1, 2**15, n_samples)
quad_q = np.random.randint(-2**15 + 1, 2**15, n_samples)
quadratures = quad_i + 1j * quad_q
series = iisr_io.TimeSeries(time_mark, parameters, quadratures)
series_list.append(series)
all_series_list.append(series)
packages_list.append(iisr_io.TimeSeriesPackage(time_mark, series_list))
with tempfile.TemporaryDirectory() as temp_dirname:
file_path = Path(temp_dirname) / DUMMY_FILE_NAME
with iisr_io.open_data_file(
file_path, 'w') as writer: # type: iisr_io.DataFileWriter
for package in packages_list:
writer.write(package)
yield file_path, all_series_list
def test_write(self):
with tempfile.TemporaryDirectory() as dirname:
path = Path(dirname) / DUMMY_FILE_NAME
with iisr_io.open_data_file(path, 'w') as writer:
self.assertTrue(path.exists())
self.assertIsInstance(writer, iisr_io.DataFileWriter)
def test_input(self):
with self.assertRaises(
ValueError), tempfile.TemporaryDirectory() as dirname:
path = Path(dirname) / DUMMY_FILE_NAME
with iisr_io.open_data_file(path, 'rb'):
pass