def test_resample(self):
raster = 1.33
sigs = [
Signal(
samples=np.arange(1000, dtype='f8'),
timestamps=np.concatenate(
[np.arange(500), np.arange(1000, 1500)]),
name=f'Signal_{i}',
) for i in range(20)
]
mdf = MDF()
mdf.append(sigs)
mdf = mdf.resample(raster=raster)
target_timestamps = np.arange(0, 1500, 1.33)
target_samples = np.concatenate([
np.arange(0, 500, 1.33),
np.linspace(499.00215568862274, 499.9976646706587, 376),
np.arange(500.1600000000001, 1000, 1.33)
])
for i, sig in enumerate(mdf.iter_channels(skip_master=True)):
self.assertTrue(np.array_equal(sig.timestamps, target_timestamps))
self.assertTrue(np.allclose(sig.samples, target_samples))
def GetDatafromMdf_asDF(filename, SigList, SampleTime=0.01, EncodeEnums=True):
## Local helper
def cleanEnumString(string):
return string.decode().split(r'\x00')[0].strip().strip('\x00')
## Function body
tempMDF = MDF()
sigs = []
with MDF(filename, remove_source_from_channel_names=True) as mdf:
for var in SigList:
try:
# get group and Index from channel_db
grp_idx = mdf.channels_db[var][0]
# Fetch signal as data object
sigs.append(mdf.get(group=grp_idx[0], index=grp_idx[1]))
# Append to mdf
except:
continue
tempMDF.append(sigs)
df = tempMDF.to_dataframe(raster=SampleTime)
types = df.apply(lambda x: pd.api.types.infer_dtype(x.values))
for col in types[types == 'bytes'].index: # String/Enum
df[col] = df[col].apply(cleanEnumString)
if (EncodeEnums == True):
types = df.apply(lambda x: pd.api.types.infer_dtype(x.values))
for col in types[types == 'string'].index: # String/Enum
df[col] = df[col].astype('category')
df[col] = df[col].cat.codes
return df
def export_to_mf4(self):
print("exporting to mf4")
timestamps = np.array(self.data['t'])
voltages = Signal(samples=np.array(self.data['v'], dtype=np.float32),
timestamps=timestamps,
name='Voltage',
unit='V')
currents = Signal(samples=np.array(self.data['c'], dtype=np.float32),
timestamps=timestamps,
name='Current',
unit='A')
powers = Signal(samples=np.array(self.data['p'], dtype=np.float32),
timestamps=timestamps,
name='Power',
unit='W')
capacities = Signal(samples=np.array(self.data['cap'],
dtype=np.float32),
timestamps=timestamps,
name='Capacity',
unit='AH')
mdf4 = MDF(version='4.10')
signals = [voltages, currents, powers, capacities]
mdf4.start_time = self.start_time
mdf4.append(signals, comment='Battery test: {}'.format(self.cell_id))
mdf4.save("test.mf4", overwrite=True)
return mdf4
def test_mixed(self):
t = np.arange(15, dtype="<f8")
s1 = Signal(
np.frombuffer(b"\x00\x00\x00\x02" * 15, dtype=">u4"), t, name="Motorola"
)
s2 = Signal(
np.frombuffer(b"\x04\x00\x00\x00" * 15, dtype="<u4"), t, name="Intel"
)
for version in ("3.30", "4.10"):
mdf = MDF(version=version)
mdf.append([s1, s2], common_timebase=True)
outfile = mdf.save(
Path(TestEndianess.tempdir.name) / f"out", overwrite=True
)
mdf.close()
with MDF(outfile) as mdf:
self.assertTrue(np.array_equal(mdf.get("Motorola").samples, [2] * 15))
self.assertTrue(np.array_equal(mdf.get("Intel").samples, [4] * 15))
for version in ("3.30", "4.10"):
mdf = MDF(version=version)
mdf.append([s2, s1], common_timebase=True)
outfile = mdf.save(
Path(TestEndianess.tempdir.name) / f"out", overwrite=True
)
mdf.close()
with MDF(outfile) as mdf:
self.assertTrue(np.array_equal(mdf.get("Motorola").samples, [2] * 15))
self.assertTrue(np.array_equal(mdf.get("Intel").samples, [4] * 15))
def test_mixed(self):
t = np.arange(15, dtype='<f8')
s1 = Signal(
np.frombuffer(b'\x00\x00\x00\x02' * 15, dtype='>u4'),
t,
name='Motorola'
)
s2 = Signal(
np.frombuffer(b'\x04\x00\x00\x00' * 15, dtype='<u4'),
t,
name='Intel'
)
for version in ('3.30', '4.10'):
mdf = MDF(version=version)
mdf.append([s1, s2], common_timebase=True)
outfile = mdf.save(
Path(TestEndianess.tempdir.name) / f"out",
overwrite=True,
)
mdf.close()
with MDF(outfile) as mdf:
self.assertTrue(
np.array_equal(
mdf.get('Motorola').samples, [2,] * 15
)
)
self.assertTrue(
np.array_equal(
mdf.get('Intel').samples, [4,] * 15
)
)
for version in ('3.30', '4.10'):
mdf = MDF(version=version)
mdf.append([s2, s1], common_timebase=True)
outfile = mdf.save(
Path(TestEndianess.tempdir.name) / f"out",
overwrite=True,
)
mdf.close()
with MDF(outfile) as mdf:
self.assertTrue(
np.array_equal(
mdf.get('Motorola').samples, [2,] * 15
)
)
self.assertTrue(
np.array_equal(
mdf.get('Intel').samples, [4,] * 15
)
)
def test_iter_groups(self):
dfs = [
DataFrame({
f'df_{i}_column_0': np.ones(5) * i,
f'df_{i}_column_1': np.arange(5) * i
}) for i in range(5)
]
mdf = MDF()
for df in dfs:
mdf.append(df)
for i, mdf_df in enumerate(mdf.iter_groups()):
self.assertTrue(mdf_df.equals(dfs[i]))
def test_resample_raster_0(self):
sigs = [
Signal(
samples=np.ones(1000) * i,
timestamps=np.arange(1000),
name=f'Signal_{i}',
) for i in range(20)
]
mdf = MDF()
mdf.append(sigs)
mdf.configure(read_fragment_size=1)
with self.assertRaises(AssertionError):
mdf = mdf.resample(raster=0)
def test_resample_raster_0(self):
sigs = [
Signal(
samples=np.ones(1000) * i,
timestamps=np.arange(1000),
name=f'Signal_{i}',
) for i in range(20)
]
mdf = MDF()
mdf.append(sigs)
mdf.configure(read_fragment_size=1)
mdf = mdf.resample(raster=0)
for i, sig in enumerate(mdf.iter_channels(skip_master=True)):
self.assertTrue(np.array_equal(sig.samples, sigs[i].samples))
self.assertTrue(np.array_equal(sig.timestamps, sigs[i].timestamps))
def export_mdf(self):
mdf = MDF()
mdf.header.start_time = self.start_time
sigs = []
cntr = 0
for name in self.signals:
if name == "Time":
continue
sigs.append(self.get(name, True))
if sigs[-1].samples.dtype.kind == "S":
sigs[-1].encoding = "utf-8"
cntr += 1
if cntr == 200:
cntr = 0
mdf.append(sigs, common_timebase=True)
sigs = []
if sigs:
mdf.append(sigs, common_timebase=True)
return mdf
def test_to_dataframe(self):
dfs = [
DataFrame({
f'df_{i}_column_0': np.ones(5) * i,
f'df_{i}_column_1': np.arange(5) * i
}) for i in range(5)
]
mdf = MDF()
for df in dfs:
mdf.append(df)
target = {}
for i in range(5):
target[f'df_{i}_column_0'] = np.ones(5) * i
target[f'df_{i}_column_1'] = np.arange(5) * i
target = DataFrame(target)
self.assertTrue(target.equals(mdf.to_dataframe()))
conversion = {'lower_{}'.format(i): i * 10 for i in range(vals)}
conversion.update({'upper_{}'.format(i): (i + 1) * 10 for i in range(vals)})
conversion.update(
{'text_{}'.format(i): 'Level {}'.format(i)
for i in range(vals)})
conversion['default'] = b'Unknown level'
sig = Signal(
6 * np.arange(cycles, dtype=np.uint64) % 240,
t,
name='Channel_value_range_to_text',
conversion=conversion,
comment='Value range to text channel',
)
sigs.append(sig)
mdf.append(sigs, 'single dimensional channels', common_timebase=True)
sigs = []
# lookup tabel with axis
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64) * 1,
np.ones((cycles, 2), dtype=np.uint64) * 2,
np.ones((cycles, 3), dtype=np.uint64) * 3,
]
types = [
('Channel_lookup_with_axis', '(2, 3)<u8'),
('channel_axis_1', '(2, )<u8'),
('channel_axis_2', '(3, )<u8'),
]
def generate_test_files(version="4.10"):
cycles = 3000
channels_count = 2000
mdf = MDF(version=version)
if version <= "3.30":
filename = r"test.mdf"
else:
filename = r"test.mf4"
if os.path.exists(filename):
return filename
t = np.arange(cycles, dtype=np.float64)
cls = v4b.ChannelConversion if version >= "4.00" else v3b.ChannelConversion
# no conversion
sigs = []
for i in range(channels_count):
sig = Signal(
np.ones(cycles, dtype=np.uint64) * i,
t,
name="Channel_{}".format(i),
unit="unit_{}".format(i),
conversion=None,
comment="Unsigned int 16bit channel {}".format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# linear
sigs = []
for i in range(channels_count):
conversion = {
"conversion_type":
v4c.CONVERSION_TYPE_LIN
if version >= "4.00" else v3c.CONVERSION_TYPE_LINEAR,
"a":
float(i),
"b":
-0.5,
}
sig = Signal(
np.ones(cycles, dtype=np.int64),
t,
name="Channel_{}".format(i),
unit="unit_{}".format(i),
conversion=cls(**conversion),
comment="Signed 16bit channel {} with linear conversion".format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# algebraic
sigs = []
for i in range(channels_count):
conversion = {
"conversion_type":
v4c.CONVERSION_TYPE_ALG
if version >= "4.00" else v3c.CONVERSION_TYPE_FORMULA,
"formula":
"{} * sin(X)".format(i),
}
sig = Signal(
np.arange(cycles, dtype=np.int32) / 100.0,
t,
name="Channel_{}".format(i),
unit="unit_{}".format(i),
conversion=cls(**conversion),
comment="Sinus channel {} with algebraic conversion".format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# rational
sigs = []
for i in range(channels_count):
conversion = {
"conversion_type":
v4c.CONVERSION_TYPE_RAT
if version >= "4.00" else v3c.CONVERSION_TYPE_RAT,
"P1":
0,
"P2":
i,
"P3":
-0.5,
"P4":
0,
"P5":
0,
"P6":
1,
}
sig = Signal(
np.ones(cycles, dtype=np.int64),
t,
name="Channel_{}".format(i),
unit="unit_{}".format(i),
conversion=cls(**conversion),
comment="Channel {} with rational conversion".format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# string
sigs = []
for i in range(channels_count):
sig = [
"Channel {} sample {}".format(i, j).encode("ascii")
for j in range(cycles)
]
sig = Signal(
np.array(sig),
t,
name="Channel_{}".format(i),
unit="unit_{}".format(i),
comment="String channel {}".format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# byte array
sigs = []
ones = np.ones(cycles, dtype=np.dtype("(8,)u1"))
for i in range(channels_count):
sig = Signal(
ones * (i % 255),
t,
name="Channel_{}".format(i),
unit="unit_{}".format(i),
comment="Byte array channel {}".format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# value to text
sigs = []
ones = np.ones(cycles, dtype=np.uint64)
conversion = {
"raw":
np.arange(255, dtype=np.float64),
"phys":
np.array(["Value {}".format(i).encode("ascii") for i in range(255)]),
"conversion_type":
v4c.CONVERSION_TYPE_TABX
if version >= "4.00" else v3c.CONVERSION_TYPE_TABX,
"links_nr":
260,
"ref_param_nr":
255,
}
for i in range(255):
conversion["val_{}".format(i)] = conversion["param_val_{}".format(
i)] = conversion["raw"][i]
conversion["text_{}".format(i)] = conversion["phys"][i]
conversion["text_{}".format(255)] = "Default"
for i in range(channels_count):
sig = Signal(
ones * i,
t,
name="Channel_{}".format(i),
unit="unit_{}".format(i),
comment="Value to text channel {}".format(i),
conversion=cls(**conversion),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
mdf.save(filename, overwrite=True)
{"upper_{}".format(i): (i + 1) * 10 - 5
for i in range(vals)})
conversion.update(
{"text_{}".format(i): "Level {}".format(i)
for i in range(vals)})
conversion["default"] = b"Unknown level"
sig = Signal(
6 * np.arange(cycles, dtype=np.uint64) % 240,
t,
name="Channel_value_range_to_text",
conversion=conversion,
comment="Value range to text channel",
)
sigs.append(sig)
mdf.append(sigs, "single dimensional channels", common_timebase=True)
sigs = []
# lookup tabel with axis
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64) * 1,
np.ones((cycles, 2), dtype=np.uint64) * 2,
np.ones((cycles, 3), dtype=np.uint64) * 3,
]
types = [
("Channel_lookup_with_axis", "(2, 3)<u8"),
("channel_axis_1", "(2, )<u8"),
("channel_axis_2", "(3, )<u8"),
]
def generate_arrays_test_file(tmpdir):
version = "4.10"
mdf = MDF(version=version)
filename = Path(tmpdir) / f"arrays_test_{version}.mf4"
if filename.exists():
return filename
t = np.arange(cycles, dtype=np.float64)
# lookup tabel with axis
sigs = []
for i in range(array_channels_count):
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64) * i,
np.ones((cycles, 2), dtype=np.uint64) * i,
np.ones((cycles, 3), dtype=np.uint64) * i,
]
types = [
("Channel_{}".format(i), "(2, 3)<u8"),
("channel_{}_axis_1".format(i), "(2, )<u8"),
("channel_{}_axis_2".format(i), "(3, )<u8"),
]
sig = Signal(
np.core.records.fromarrays(samples, dtype=np.dtype(types)),
t,
name="Channel_{}".format(i),
unit="unit_{}".format(i),
conversion=None,
comment="Array channel {}".format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# lookup tabel with default axis
sigs = []
for i in range(array_channels_count):
samples = [np.ones((cycles, 2, 3), dtype=np.uint64) * i]
types = [("Channel_{}".format(i), "(2, 3)<u8")]
sig = Signal(
np.core.records.fromarrays(samples, dtype=np.dtype(types)),
t,
name="Channel_{}".format(i),
unit="unit_{}".format(i),
conversion=None,
comment="Array channel {} with default axis".format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# structure channel composition
sigs = []
for i in range(array_channels_count):
samples = [
np.ones(cycles, dtype=np.uint8) * i,
np.ones(cycles, dtype=np.uint16) * i,
np.ones(cycles, dtype=np.uint32) * i,
np.ones(cycles, dtype=np.uint64) * i,
np.ones(cycles, dtype=np.int8) * i,
np.ones(cycles, dtype=np.int16) * i,
np.ones(cycles, dtype=np.int32) * i,
np.ones(cycles, dtype=np.int64) * i,
]
types = [
("struct_{}_channel_0".format(i), np.uint8),
("struct_{}_channel_1".format(i), np.uint16),
("struct_{}_channel_2".format(i), np.uint32),
("struct_{}_channel_3".format(i), np.uint64),
("struct_{}_channel_4".format(i), np.int8),
("struct_{}_channel_5".format(i), np.int16),
("struct_{}_channel_6".format(i), np.int32),
("struct_{}_channel_7".format(i), np.int64),
]
sig = Signal(
np.core.records.fromarrays(samples, dtype=np.dtype(types)),
t,
name="Channel_{}".format(i),
unit="unit_{}".format(i),
conversion=None,
comment="Structure channel composition {}".format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
name = mdf.save(filename, overwrite=True)
timestamps=timestamps,
name='Int32_Signal',
unit='i4')
# float64
s_float64 = Signal(samples=np.array([-20, -10, 0, 10, 20], dtype=np.float64),
timestamps=timestamps,
name='Float64_Signal',
unit='f8')
# create empty MDf version 4.00 file
mdf4 = MDF(version='4.10')
# append the 3 signals to the new file
signals = [s_uint8, s_int32, s_float64]
mdf4.append(signals, 'Created by Python')
# save new file
mdf4.save('my_new_file.mf4', overwrite=True)
# convert new file to mdf version 3.10 with lowest possible RAM usage
mdf3 = mdf4.convert('3.10', memory='minimum')
print(mdf3.version)
# get the float signal
sig = mdf3.get('Float64_Signal')
print(sig)
# cut measurement from 0.3s to end of measurement
mdf4_cut = mdf4.cut(start=0.3)
mdf4_cut.get('Float64_Signal').plot()
def generate_test_files():
print('Generating test files:')
for version in ('3.30', '4.10'):
print("-> generating file for version", version)
mdf = MDF(version=version, memory='minimum')
if version == '3.30':
cycles = 500
channels_count = 8000
filename = 'test.mdf'
else:
cycles = 500
channels_count = 20000
filename = 'test.mf4'
if os.path.exists(filename):
continue
t = np.arange(cycles, dtype=np.float64)
# no conversion
sigs = []
for i in range(channels_count):
sig = Signal(
np.ones(cycles, dtype=np.uint16),
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=None,
comment='Unsinged int 16bit channel {}'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs)
# linear
sigs = []
for i in range(channels_count):
conversion = {
'type': SignalConversions.CONVERSION_LINEAR,
'a': float(i),
'b': -0.5,
}
sig = Signal(
np.ones(cycles, dtype=np.int16),
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=conversion,
comment='Signed 16bit channel {} with linear conversion'.
format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs)
# algebraic
sigs = []
for i in range(channels_count):
conversion = {
'type': SignalConversions.CONVERSION_ALGEBRAIC,
'formula': '{} * sin(X)'.format(i),
}
sig = Signal(
np.arange(cycles, dtype=np.int32) / 100,
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=conversion,
comment='Sinus channel {} with algebraic conversion'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs)
# rational
sigs = []
for i in range(channels_count):
conversion = {
'type': SignalConversions.CONVERSION_RATIONAL,
'P1': 0,
'P2': i,
'P3': -0.5,
'P4': 0,
'P5': 0,
'P6': 1,
}
sig = Signal(
np.ones(cycles, dtype=np.int64),
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=conversion,
comment='Channel {} with rational conversion'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs)
mdf.save(filename, overwrite=True)
del mdf
MDF.merge([
filename,
] * 10, version, memory='minimum').save(filename, overwrite=True)
def generate_test_files(version='4.10'):
cycles = 3000
channels_count = 2000
mdf = MDF(version=version)
if version <= '3.30':
filename = r'test.mdf'
else:
filename = r'test.mf4'
if os.path.exists(filename):
return filename
t = np.arange(cycles, dtype=np.float64)
cls = v4b.ChannelConversion if version >= '4.00' else v3b.ChannelConversion
# no conversion
sigs = []
for i in range(channels_count):
sig = Signal(
np.ones(cycles, dtype=np.uint64) * i,
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=None,
comment='Unsigned int 16bit channel {}'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# linear
sigs = []
for i in range(channels_count):
conversion = {
'conversion_type':
v4c.CONVERSION_TYPE_LIN
if version >= '4.00' else v3c.CONVERSION_TYPE_LINEAR,
'a':
float(i),
'b':
-0.5,
}
sig = Signal(
np.ones(cycles, dtype=np.int64),
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=cls(**conversion),
comment='Signed 16bit channel {} with linear conversion'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# algebraic
sigs = []
for i in range(channels_count):
conversion = {
'conversion_type':
v4c.CONVERSION_TYPE_ALG
if version >= '4.00' else v3c.CONVERSION_TYPE_FORMULA,
'formula':
'{} * sin(X)'.format(i),
}
sig = Signal(
np.arange(cycles, dtype=np.int32) / 100.0,
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=cls(**conversion),
comment='Sinus channel {} with algebraic conversion'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# rational
sigs = []
for i in range(channels_count):
conversion = {
'conversion_type':
v4c.CONVERSION_TYPE_RAT
if version >= '4.00' else v3c.CONVERSION_TYPE_RAT,
'P1':
0,
'P2':
i,
'P3':
-0.5,
'P4':
0,
'P5':
0,
'P6':
1,
}
sig = Signal(
np.ones(cycles, dtype=np.int64),
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=cls(**conversion),
comment='Channel {} with rational conversion'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# string
sigs = []
for i in range(channels_count):
sig = [
'Channel {} sample {}'.format(i, j).encode('ascii')
for j in range(cycles)
]
sig = Signal(
np.array(sig),
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
comment='String channel {}'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# byte array
sigs = []
ones = np.ones(cycles, dtype=np.dtype('(8,)u1'))
for i in range(channels_count):
sig = Signal(
ones * (i % 255),
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
comment='Byte array channel {}'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
# value to text
sigs = []
ones = np.ones(cycles, dtype=np.uint64)
conversion = {
'raw':
np.arange(255, dtype=np.float64),
'phys':
np.array(['Value {}'.format(i).encode('ascii') for i in range(255)]),
'conversion_type':
v4c.CONVERSION_TYPE_TABX
if version >= '4.00' else v3c.CONVERSION_TYPE_TABX,
'links_nr':
260,
'ref_param_nr':
255,
}
for i in range(255):
conversion['val_{}'.format(i)] = conversion['param_val_{}'.format(
i)] = conversion['raw'][i]
conversion['text_{}'.format(i)] = conversion['phys'][i]
conversion['text_{}'.format(255)] = 'Default'
for i in range(channels_count):
sig = Signal(
ones * i,
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
comment='Value to text channel {}'.format(i),
conversion=cls(**conversion),
raw=True,
)
sigs.append(sig)
mdf.append(sigs, common_timebase=True)
mdf.save(filename, overwrite=True)
