def test_files_list(self):
patterns = list(TEST_FILE_PATTERNS.keys())
d = DecadesDataset()
for pattern in patterns:
d.add_file(pattern)
for pattern in patterns:
self.assertIn(pattern, [i.filepath for i in d.files])
def setUp(self):
self.test_index_1 = pd.date_range(start=datetime.datetime(2000, 1, 1),
periods=10,
freq='S')
self.test_index_32 = pd.date_range(start=datetime.datetime(2000, 1, 1),
periods=320,
freq='31250000N')
self.d = DecadesDataset(datetime.datetime(2000, 1, 1))
self.v = DecadesVariable({TEST_VAR_NAME: TEST_VAR_VALUES},
index=self.test_index_1,
frequency=1)
def test_add_decades_file(self):
patterns = list(TEST_FILE_PATTERNS.keys())
d = DecadesDataset()
for pattern in patterns:
d.add_file(pattern)
files = []
for i in d.readers:
files += i.files
for _file in files:
self.assertIn(_file.filepath, patterns)
def test_instance(cls, dataset=None):
"""
Return a test instance of a postprocessing module, initialized with a
DecadesDataset containing the modules test data.
"""
now = datetime.datetime.now().replace(microsecond=0)
if dataset is None:
d = DecadesDataset(now.date())
else:
d = dataset
if callable(cls.test):
# pylint: disable=not-callable
_test = cls.test()
else:
_test = cls.test
for key, val in _test.items():
_type, *_values = val
if val[0] == 'const':
d.constants[key] = val[1]
elif val[0] == 'data':
_values, _freq = val[1:]
_dt = datetime.timedelta(seconds=1 / _freq)
freq = '{}N'.format((1 / _freq) * 1e9)
start_time = datetime.datetime(*d.date.timetuple()[:3])
end_time = (start_time +
datetime.timedelta(seconds=len(_values)) - _dt)
hz1_index = pd.date_range(start=start_time,
periods=len(_values),
freq='S')
full_index = pd.date_range(start=start_time,
end=end_time,
freq=freq)
data = pd.Series(_values,
hz1_index).reindex(full_index).interpolate()
var = DecadesVariable(data, name=key, frequency=_freq)
d.add_input(var)
return cls(d, test_mode=True)
def test_infer_readers(self):
for pattern, reader in TEST_FILE_PATTERNS.items():
self.assertIs(DecadesDataset.infer_reader(DecadesFile(pattern)),
reader)
os.remove(pattern)
def get_flagmod_doc(module):
"""
Returns documentation for a given flagging module, as restructured text.
Args:
module: the flagging module to document, expected to be a subclass of
ppodd.flags.FlaggingBase.
Returns:
A restructured text string containing the module documentation.
"""
index = module.test_index
flag = module.test_flag
d = DecadesDataset()
for var in module.flagged:
v = DecadesVariable(pd.Series(flag, index=index, name=var),
flag=DecadesBitmaskFlag)
d.add_output(v)
mod = module(d)
mod._flag(test=True)
output = "\n\n"
output += '-' * len(module.__name__) + '\n'
output += module.__name__ + '\n'
output += '-' * len(module.__name__) + '\n\n'
output += mod.__doc__
for var, flag_infos in mod.flags.items():
mod_var_txt = f'Flagged variable: `{var}`'
output += '\n\n' + mod_var_txt + '\n'
output += '-' * len(mod_var_txt) + '\n\n'
for flag_info in flag_infos:
output += f'* ``{flag_info[0]}`` - {flag_info[1]}\n'
return output
def test_dataset_date_attribute(self):
today = datetime.date.today()
d = DecadesDataset(today)
self.assertEqual(d.date, today)
def test_create_dataset_with_date(self):
d = DecadesDataset(datetime.date.today())
def test_create_dataset(self):
d = DecadesDataset()
class TestDecades(unittest.TestCase):
"""
An empty subclass of unittest.TestCase, to which we append tests for each
module.
"""
def setUp(self):
self.test_index_1 = pd.date_range(start=datetime.datetime(2000, 1, 1),
periods=10,
freq='S')
self.test_index_32 = pd.date_range(start=datetime.datetime(2000, 1, 1),
periods=320,
freq='31250000N')
self.d = DecadesDataset(datetime.datetime(2000, 1, 1))
self.v = DecadesVariable({TEST_VAR_NAME: TEST_VAR_VALUES},
index=self.test_index_1,
frequency=1)
@classmethod
def setUpClass(cls):
for pattern in TEST_FILE_PATTERNS.keys():
with open(pattern, 'w'):
pass
@classmethod
def teardownClass(cls):
for pattern in TEST_FILE_PATTERNS.keys():
os.remove(pattern)
def _get_var_1(self):
return DecadesVariable({TEST_VAR_NAME: TEST_VAR_VALUES},
index=self.test_index_1)
def _get_var_32(self):
return DecadesVariable({TEST_VAR_NAME: TEST_VAR_VALUES * 32},
index=self.test_index_32)
def test_create_dataset(self):
d = DecadesDataset()
def test_variable_isnumeric(self):
self.assertTrue(self.v.isnumeric)
v2 = DecadesVariable({'stringvar': ['a'] * len(TEST_VAR_VALUES)},
index=self.test_index_1,
frequency=1)
self.assertFalse(v2.isnumeric)
def test_create_dataset_with_date(self):
d = DecadesDataset(datetime.date.today())
def test_create_decades_variable(self):
self._get_var_1()
def test_t0_correct(self):
v = self._get_var_1()
self.assertEqual(v.t0, self.test_index_1[0])
def test_t1_correct(self):
v = self._get_var_1()
self.assertEqual(v.t1, self.test_index_1[-1])
def test_time_bounds(self):
v = self._get_var_1()
i = self.test_index_1
bounds = v.time_bounds()
self.assertEqual(bounds[0], self.test_index_1[0])
self.assertEqual(bounds[1], self.test_index_1[-1])
def test_dataset_date_attribute(self):
today = datetime.date.today()
d = DecadesDataset(today)
self.assertEqual(d.date, today)
def test_calling_variable_name(self):
v = self._get_var_1()()
self.assertEqual(v.name, TEST_VAR_NAME)
def test_calling_variable_index(self):
v = self._get_var_1()()
for a, b in zip(v.index, self.test_index_1):
self.assertEqual(a, b)
def test_get_freq_for_1hz_and_32hz(self):
var1 = self._get_var_1()
var32 = self._get_var_32()
self.assertEqual(var1.frequency, 1)
self.assertEqual(var32.frequency, 32)
def test_calling_variable_values(self):
v = self._get_var_1()()
for a, b in zip(v.values, TEST_VAR_VALUES):
self.assertEqual(a, b)
def test_variable_array_attribute(self):
v = self._get_var_1()
for a, b in zip(v.array, TEST_VAR_VALUES):
self.assertEqual(a, b)
def test_variable_length(self):
v = self._get_var_1()
self.assertEqual(len(v), len(v.array))
def test_variable_trim(self):
v = self._get_var_1()
start = self.test_index_1[3]
end = self.test_index_1[-3]
v.trim(start, end)
self.assertEqual(v.t0, start)
self.assertEqual(v.t1, end)
f = v.flag()
self.assertEqual(f.index[0], start)
self.assertEqual(f.index[-1], end)
def test_recover_index(self):
v = self._get_var_1()
index = v.index
self.d.add_input(v)
for a, b in zip(self.d[v.name].index, index):
self.assertEqual(a, b)
def test_variable_merge_contiguous(self):
index1 = pd.date_range(start=datetime.datetime(2000, 1, 1),
periods=10,
freq='S')
index2 = pd.date_range(start=datetime.datetime(2000, 1, 1, 0, 0, 10),
periods=10,
freq='S')
v1 = DecadesVariable({TEST_VAR_NAME: TEST_VAR_VALUES},
index=index1,
frequency=1)
v2 = DecadesVariable({TEST_VAR_NAME: TEST_VAR_VALUES},
index=index2,
frequency=1)
v1.merge(v2)
self.assertEqual(v1.t0, index1[0])
self.assertEqual(v1.t1, index2[-1])
self.assertEqual(len(v1), 2 * (len(TEST_VAR_VALUES)))
def test_variable_merge_noncontiguous(self):
index1 = pd.date_range(start=datetime.datetime(2000, 1, 1),
periods=10,
freq='S')
index2 = pd.date_range(start=datetime.datetime(2000, 1, 1, 0, 0, 15),
periods=10,
freq='S')
v1 = DecadesVariable({TEST_VAR_NAME: TEST_VAR_VALUES},
index=index1,
frequency=1)
v2 = DecadesVariable({TEST_VAR_NAME: TEST_VAR_VALUES},
index=index2,
frequency=1)
v1.merge(v2)
self.assertEqual(v1.t0, index1[0])
self.assertEqual(v1.t1, index2[-1])
self.assertEqual(len(v1), 2 * (len(TEST_VAR_VALUES)) + 5)
def test_variable_merge_interleaved(self):
index1 = pd.date_range(start=datetime.datetime(2000, 1, 1),
periods=10,
freq='2S')
index2 = pd.date_range(start=datetime.datetime(2000, 1, 1, 0, 0, 1),
periods=10,
freq='2S')
v1 = DecadesVariable({TEST_VAR_NAME: TEST_VAR_VALUES},
index=index1,
frequency=1)
v2 = DecadesVariable({TEST_VAR_NAME: TEST_VAR_VALUES},
index=index2,
frequency=1)
v1.merge(v2)
self.assertEqual(v1.t0, index1[0])
self.assertEqual(v1.t1, index2[-1])
self.assertEqual(len(v1), 2 * (len(TEST_VAR_VALUES)))
expected_array = []
for i in TEST_VAR_VALUES:
expected_array.append(i)
expected_array.append(i)
for a, b in zip(v1.array, expected_array):
self.assertEqual(a, b)
def test_infer_readers(self):
for pattern, reader in TEST_FILE_PATTERNS.items():
self.assertIs(DecadesDataset.infer_reader(DecadesFile(pattern)),
reader)
os.remove(pattern)
def test_files_list(self):
patterns = list(TEST_FILE_PATTERNS.keys())
d = DecadesDataset()
for pattern in patterns:
d.add_file(pattern)
for pattern in patterns:
self.assertIn(pattern, [i.filepath for i in d.files])
def test_add_decades_file(self):
patterns = list(TEST_FILE_PATTERNS.keys())
d = DecadesDataset()
for pattern in patterns:
d.add_file(pattern)
files = []
for i in d.readers:
files += i.files
for _file in files:
self.assertIn(_file.filepath, patterns)
def test_takeoff_land_time(self):
vals = [1, 1, 1, 1, 0, 0, 0, 1, 1, 1]
prtaft = DecadesVariable({'PRTAFT_wow_flag': vals},
index=self.test_index_1)
self.d.add_input(prtaft)
self.assertEqual(self.d.takeoff_time, self.test_index_1[4])
self.assertEqual(self.d.landing_time, self.test_index_1[7])
def test_dataset_time_bounds_inputs(self):
v1 = DecadesVariable({'a': TEST_VAR_VALUES}, index=self.test_index_1)
index2 = self.test_index_1 + datetime.timedelta(minutes=1)
v2 = DecadesVariable({'b': TEST_VAR_VALUES}, index=index2)
self.d.add_input(v1)
self.d.add_input(v2)
bnds = self.d.time_bounds()
self.assertEqual(bnds[0], self.test_index_1[0])
self.assertEqual(bnds[1], index2[-1])
def test_dataset_time_bounds_outputs(self):
v1 = DecadesVariable({'a': TEST_VAR_VALUES},
index=self.test_index_1,
frequency=1)
index2 = self.test_index_1 + datetime.timedelta(minutes=1)
v2 = DecadesVariable({'b': TEST_VAR_VALUES}, index=index2)
self.d.add_output(v1)
self.d.add_output(v2)
bnds = self.d.time_bounds()
self.assertEqual(bnds[0], self.test_index_1[0])
self.assertEqual(bnds[1], index2[-1])
def test_dataset_time_bounds_input_output(self):
v1 = DecadesVariable({'a': TEST_VAR_VALUES},
index=self.test_index_1,
frequency=1)
index2 = self.test_index_1 + datetime.timedelta(minutes=1)
v2 = DecadesVariable({'b': TEST_VAR_VALUES}, index=index2, frequency=1)
self.d.add_output(v1)
self.d.add_input(v2)
bnds = self.d.time_bounds()
self.assertEqual(bnds[0], self.test_index_1[0])
self.assertEqual(bnds[1], index2[-1])
def test_add_input_variable(self):
self.d.add_input(self.v)
self.assertIn(self.v, self.d._backend.inputs)
def test_add_output_variable(self):
self.d.add_output(self.v)
self.assertIn(self.v, self.d._backend.outputs)
def test_list_variables(self):
v1 = DecadesVariable({'a': TEST_VAR_VALUES},
index=self.test_index_1,
frequency=1)
v2 = DecadesVariable({'b': TEST_VAR_VALUES},
index=self.test_index_1,
frequency=1)
self.d.add_input(v1)
self.d.add_output(v2)
self.assertIn(v1.name, self.d.variables)
self.assertIn(v2.name, self.d.variables)
def test_remove_variable(self):
self.d.add_input(self.v)
self.d.remove(self.v.name)
self.assertEqual(self.d.variables, [])
self.d.add_output(self.v)
self.d.remove(self.v.name)
self.assertEqual(self.d.variables, [])
def test_add_constant(self):
self.d.add_constant('LTUAE', 42)
self.assertEqual(self.d['LTUAE'], 42)
def test_lazy_constant_item(self):
value = self.d.lazy['LTUAE']
self.d.add_constant('LTUAE', 42)
self.assertEqual(value(), 42)
self.assertEqual(self.d.lazy['LTUAE'], 42)
def test_outputs_list(self):
v1 = self.v
v2 = DecadesVariable({'v2': TEST_VAR_VALUES},
index=self.test_index_1,
frequency=1)
self.d.add_output(v1)
self.d.add_output(v2)
self.assertIn(v1, self.d.outputs)
self.assertIn(v2, self.d.outputs)
def test_clear_outputs(self):
v1 = self.v
v2 = DecadesVariable({'v2': TEST_VAR_VALUES},
index=self.test_index_1,
frequency=1)
self.d.add_output(v1)
self.d.add_output(v2)
self.d.clear_outputs()
self.assertEqual(self.d.outputs, [])
def test_add_global(self):
self.assertRaises(NonStandardAttributeError,
lambda: self.d.add_global('LTUAE', 42))
self.d.globals.strict = False
self.d.add_global('LTUAE', 42)
self.assertEqual(self.d.globals()['LTUAE'], 42)
def test_add_interpolated_global(self):
self.d.globals.strict = False
self.d.add_global('one', 2)
self.d.add_global('two', '{one}')
self.d._interpolate_globals()
self.assertEqual(self.d.globals()['two'], '2')
def test_add_data_global(self):
self.d.add_output(self.v)
self.d.globals.strict = False
self.d.add_global('test1', f'<data {TEST_VAR_NAME} max>')
self.assertEqual(self.d.globals()['test1'], max(TEST_VAR_VALUES))
self.d.add_global('test2', f'<call datetime.date.today>')
self.assertEqual(self.d.globals()['test2'], datetime.date.today())