def test_9500(self):
parsed_data = timeseries.parse_timeseries(data=self.e_cons_meas_9500)
# For this data, we have 6 measurements because the platform
# outputs data every 3 seconds, and we ran a 20 second
# simulation.
self.assertEqual(6, parsed_data.shape[0])
# With just one measurement, we shouldn't get any nans.
self.assertFalse(parsed_data.isna().any().any())
# Ensure we have the expected columns.
self.assertListEqual(
['hasSimulationMessageType', 'measurement_mrid', 'angle',
'magnitude', 'simulation_id'],
parsed_data.columns.to_list()
)
# Ensure all our times are in bounds.
self.assertTrue((parsed_data.index >= self.starttime).all())
self.assertTrue((parsed_data.index <= self.stoptime).all())
# Ensure all columns which should be numeric are numeric.
for c in parsed_data.columns.to_list():
if c in NUMERIC_COLS:
self.assertEqual(np.dtype('float'), parsed_data[c].dtype)
def generate_parsed_sensor_service_measurements_9500():
"""Given the data generated by
generate_sensor_service_measurements_9500, parse them and save the
results.
"""
for idx, file in enumerate(SENSOR_MEAS_LIST):
with open(file, 'r') as f:
data = json.load(f)
parsed_data = timeseries.parse_timeseries(data)
to_file(parsed_data, PARSED_SENSOR_LIST[idx])
def test_13_not_indexed_by_time(self):
"""Ensure the index_by_time parameter is working as intended."""
parsed = timeseries.parse_timeseries(data=self.meas_13,
index_by_time=False)
# In this case, we should have 'time' as a column rather than
# the index.
self.assertIn('time', parsed.columns)
# Ensure we just have a simple range index.
self.assertEqual(parsed.index[0], 0)
self.assertEqual(parsed.index[-1], parsed.shape[0] - 1)
def test_13(self):
# Parse the data.
parsed_data = timeseries.parse_timeseries(data=self.meas_13)
# We should get back as many rows as there are entries in 'data'
self.assertEqual(len(self.meas_13['data']), parsed_data.shape[0])
# For the 13 node "all measurements" we'll be getting back
# mixed types which results in columns which are not used for
# all the data. Ensure we have NaN's.
self.assertTrue(
parsed_data.isna().any().any()
)
# Ensure we have the expected columns.
self.assertSetEqual(
{'hasSimulationMessageType', 'measurement_mrid', 'simulation_id',
'angle', 'magnitude', 'value'},
set(parsed_data.columns.to_list())
)
# The NA signature should be identical for angle and magnitude.
angle_na = parsed_data['angle'].isna()
mag_na = parsed_data['magnitude'].isna()
pd.testing.assert_series_equal(angle_na, mag_na,
check_names=False)
# The NA signature should be exactly opposite for
# angle/magnitude and value.
value_na = parsed_data['value'].isna()
xor = np.logical_xor(angle_na.values, value_na.values)
self.assertTrue(xor.all())
# Ensure all our times are in bounds.
self.assertTrue((parsed_data.index >= self.starttime).all())
self.assertTrue((parsed_data.index <= self.stoptime).all())
# Ensure all columns which should be numeric are numeric.
for c in parsed_data.columns.to_list():
if c in NUMERIC_COLS:
self.assertEqual(np.dtype('float'), parsed_data[c].dtype)
def get_simulation_output(
self, simulation_id, query_measurement='simulation',
starttime=None, endtime=None,
measurement_mrid: Union[List[str], str, None] = None,
index_by_time=True):
"""Simple wrapper to call _query_simulation_output and then
parse and return the results. See the docstring of
_query_simulation_output for details on inputs.
TODO: document parameters.
:param index_by_time: Passed to timeseries.parse_timeseries.
"""
# Query the timeseries database.
data = \
self._query_simulation_output(
simulation_id=simulation_id,
query_measurement=query_measurement, starttime=starttime,
endtime=endtime, measurement_mrid=measurement_mrid)
# Parse the result and return.
return timeseries.parse_timeseries(data, index_by_time)
def test_sensor_service_9500(self):
# Read in data.
data = []
for file in _df.SENSOR_MEAS_LIST:
with open(file, 'r') as f:
data.append(json.load(f))
# Parse each dictionary and do some rudimentary tests. Mainly,
# we're happy if it just parses.
for d in data:
parsed = timeseries.parse_timeseries(data=d)
# Ensure it's a DataFrame.
self.assertIsInstance(parsed, pd.DataFrame)
# Ensure we're indexed by time.
self.assertEqual('time', parsed.index.name)
self.assertIsInstance(parsed.index[0], datetime)
# Ensure we have all the expected columns.
self.assertListEqual(
['instance_id', 'hasSimulationMessageType', 'measurement_mrid',
'angle', 'magnitude', 'simulation_id'],
parsed.columns.to_list()
)
# Ensure angle and magnitude are floats.
self.assertEqual(np.dtype('float'), parsed.dtypes['angle'])
self.assertEqual(np.dtype('float'), parsed.dtypes['magnitude'])
# The rest of our columns should be objects.
cols = copy.copy(parsed.columns.to_list())
cols.remove('angle')
cols.remove('magnitude')
for c in cols:
self.assertEqual(np.dtype('O'), parsed.dtypes[c])