def merge_v4(output, fmt):
files = [
r'test.mf4',
] * 3
with Timer('Merge 3 files', f'asammdf {asammdf_version} v4', fmt) as timer:
MDF.concatenate(files, outversion='4.10')
output.send([timer.output, timer.error])
def merge_v4(output, fmt):
files = [
r"test.mf4",
] * 3
with Timer("Merge 3 files", f"asammdf {asammdf_version} v4", fmt) as timer:
MDF.concatenate(files, outversion="4.10")
output.send([timer.output, timer.error])
def test_cut_demo(self):
print("MDF cut demo tests")
mdf_files = [
file for file in Path(TestMDF.tempdir_demo.name).iterdir()
if file.suffix in ('.mdf', '.mf4')
]
signals = [
file for file in Path(TestMDF.tempdir_demo.name).iterdir()
if file.suffix == '.npy'
]
for file in mdf_files:
print(file)
for inp in (file, BytesIO(file.read_bytes())):
with MDF(inp, use_display_names=True) as input_file:
for whence in (0, 1):
print(file, whence)
outfile1 = (input_file.cut(
stop=2, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp1",
overwrite=True))
outfile2 = (input_file.cut(
start=2, stop=6, whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp2",
overwrite=True))
outfile3 = (input_file.cut(
start=6, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp3",
overwrite=True))
outfile = MDF.concatenate(
[outfile1, outfile2, outfile3],
version=input_file.version,
use_display_names=True,
).save(Path(TestMDF.tempdir.name) / "tmp",
overwrite=True)
print("OUT", outfile)
with MDF(outfile, use_display_names=True) as mdf2:
for signal in signals:
target = np.load(signal)
sig = mdf2.get(signal.stem)
timestamps = input_file.get(
signal.stem).timestamps
self.assertTrue(
np.array_equal(sig.samples, target))
self.assertTrue(
np.array_equal(timestamps, sig.timestamps))
def test_cut_demo(self):
print("MDF cut absolute tests")
cntr = 0
for mdfname in os.listdir("tmpdir_demo"):
for memory in MEMORY:
input_file = os.path.join("tmpdir_demo", mdfname)
if "2.00" in input_file:
continue
for whence in (0, 1):
print(input_file, memory, whence)
outfile1 = (MDF(input_file, memory=memory).cut(
stop=2, whence=whence,
include_ends=False).save("tmp1", overwrite=True))
outfile2 = (MDF(input_file, memory=memory).cut(
start=2, stop=6, whence=whence,
include_ends=False).save("tmp2", overwrite=True))
outfile3 = (MDF(input_file, memory=memory).cut(
start=6, whence=whence,
include_ends=False).save("tmp3", overwrite=True))
outfile = MDF.concatenate(
[outfile1, outfile2, outfile3],
vedrsion=MDF(input_file, memory="minimum").version,
memory=memory,
).save("tmp", overwrite=True)
print("OUT", outfile)
equal = True
with MDF(input_file,
memory=memory) as mdf, MDF(outfile,
memory=memory) as mdf2:
for i, group in enumerate(mdf.groups):
for j, _ in enumerate(group["channels"][1:], 1):
original = mdf.get(group=i, index=j)
converted = mdf2.get(group=i, index=j)
if not np.array_equal(original.samples,
converted.samples):
equal = False
if not np.array_equal(original.timestamps,
converted.timestamps):
equal = False
self.assertTrue(equal)
cleanup_files()
def test_cut_demo(self):
print("MDF cut absolute tests")
cntr = 0
for mdfname in os.listdir('tmpdir_demo'):
for memory in MEMORY:
input_file = os.path.join('tmpdir_demo', mdfname)
if '2.00' in input_file:
continue
for whence in (0, 1):
print(input_file, memory, whence)
outfile1 = MDF(input_file, memory=memory).cut(
stop=2, whence=whence).save('tmp1', overwrite=True)
outfile2 = MDF(input_file, memory=memory).cut(
start=2, stop=6, whence=whence).save('tmp2',
overwrite=True)
outfile3 = MDF(input_file, memory=memory).cut(
start=6, whence=whence).save('tmp3', overwrite=True)
outfile = MDF.concatenate(
[outfile1, outfile2, outfile3],
vedrsion=MDF(input_file, memory='minimum').version,
memory=memory,
).save('tmp', overwrite=True)
print('OUT', outfile)
equal = True
with MDF(input_file, memory=memory) as mdf, \
MDF(outfile, memory=memory) as mdf2:
for i, group in enumerate(mdf.groups):
for j, _ in enumerate(group['channels'][1:], 1):
original = mdf.get(group=i, index=j)
converted = mdf2.get(group=i, index=j)
if not np.array_equal(original.samples,
converted.samples):
equal = False
if not np.array_equal(original.timestamps,
converted.timestamps):
equal = False
self.assertTrue(equal)
cleanup_files()
def test_cut_demo(self):
print("MDF cut absolute tests")
for input_file in Path(TestMDF.tempdir_demo.name).iterdir():
if "2.00" in input_file.name:
continue
for whence in (0, 1):
print(input_file, whence)
outfile1 = (MDF(input_file).cut(
stop=2, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp1", overwrite=True))
outfile2 = (MDF(input_file).cut(
start=2, stop=6, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp2", overwrite=True))
outfile3 = (MDF(input_file).cut(
start=6, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp3", overwrite=True))
outfile = MDF.concatenate(
[outfile1, outfile2, outfile3],
version=MDF(input_file).version).save(
Path(TestMDF.tempdir.name) / "tmp", overwrite=True)
print("OUT", outfile)
equal = True
with MDF(input_file) as mdf, MDF(outfile) as mdf2:
for i, group in enumerate(mdf.groups):
for j, _ in enumerate(group["channels"][1:], 1):
original = mdf.get(group=i, index=j)
converted = mdf2.get(group=i, index=j)
if not np.array_equal(original.samples,
converted.samples):
equal = False
if not np.array_equal(original.timestamps,
converted.timestamps):
equal = False
self.assertTrue(equal)
def test_cut_arrays(self):
print("MDF cut big array files")
for input_file in Path(TestMDF.tempdir_array.name).iterdir():
for whence in (0, 1):
print(input_file, whence)
outfile1 = MDF(input_file)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(
stop=105.5, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp1", overwrite=True)
outfile2 = MDF(input_file)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105.5,
stop=201.5,
whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) /
"tmp2",
overwrite=True)
outfile3 = MDF(input_file)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(
start=201.5, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp3", overwrite=True)
outfile = MDF.concatenate(
[outfile1, outfile2, outfile3],
MDF(input_file).version).save(Path(TestMDF.tempdir.name) /
"tmp_cut",
overwrite=True)
equal = True
with MDF(outfile) as mdf:
mdf.configure(read_fragment_size=8000)
for i, group in enumerate(mdf.groups):
if i == 0:
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64),
np.ones((cycles, 2), dtype=np.uint64),
np.ones((cycles, 3), dtype=np.uint64),
]
for j in range(1, 20):
types = [
("Channel_{}".format(j), "(2, 3)<u8"),
("channel_{}_axis_1".format(j),
"(2, )<u8"),
("channel_{}_axis_2".format(j),
"(3, )<u8"),
]
types = np.dtype(types)
vals = mdf.get(
"Channel_{}".format(j),
group=i,
samples_only=True,
)[0]
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
print(
i,
j,
len(target),
len(vals),
vals,
target,
sep="\n\n",
)
1 / 0
elif i == 1:
samples = np.ones((cycles, 2, 3), dtype=np.uint64)
axis_0 = np.ones((cycles, 2), dtype=np.uint64)
axis_1 = np.ones((cycles, 3), dtype=np.uint64)
for j in range(1, 20):
types = [("Channel_{}".format(j), "(2, 3)<u8")]
types = np.dtype(types)
vals = mdf.get(
"Channel_{}".format(j),
group=i,
samples_only=True,
)[0]
target = [samples * j]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
1 / 0
elif i == 2:
samples = [
np.ones(cycles, dtype=np.uint8),
np.ones(cycles, dtype=np.uint16),
np.ones(cycles, dtype=np.uint32),
np.ones(cycles, dtype=np.uint64),
np.ones(cycles, dtype=np.int8),
np.ones(cycles, dtype=np.int16),
np.ones(cycles, dtype=np.int32),
np.ones(cycles, dtype=np.int64),
]
for j in range(1, 20):
types = [
("struct_{}_channel_0".format(j),
np.uint8),
("struct_{}_channel_1".format(j),
np.uint16),
("struct_{}_channel_2".format(j),
np.uint32),
("struct_{}_channel_3".format(j),
np.uint64),
("struct_{}_channel_4".format(j), np.int8),
("struct_{}_channel_5".format(j),
np.int16),
("struct_{}_channel_6".format(j),
np.int32),
("struct_{}_channel_7".format(j),
np.int64),
]
types = np.dtype(types)
vals = mdf.get(
"Channel_{}".format(j),
group=i,
samples_only=True,
)[0]
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
1 / 0
self.assertTrue(equal)
def test_cut(self):
print("MDF cut big files tests")
t = np.arange(cycles, dtype=np.float64)
for input_file in Path(TestMDF.tempdir_general.name).iterdir():
for whence in (0, 1):
for compression in range(3):
print(input_file)
outfile0 = MDF(input_file)
outfile0.configure(read_fragment_size=8000)
outfile0 = outfile0.cut(stop=-1,
whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) /
"tmp0",
overwrite=True,
compression=compression,
)
outfile1 = MDF(input_file)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105,
whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) /
"tmp1",
overwrite=True,
compression=compression,
)
outfile2 = MDF(input_file)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105.1,
stop=201,
whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) /
"tmp2",
overwrite=True,
compression=compression,
)
outfile3 = MDF(input_file)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(
start=201.1, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp3",
overwrite=True)
outfile4 = MDF(input_file)
outfile4.configure(read_fragment_size=8000)
outfile4 = outfile4.cut(start=7000,
whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) /
"tmp4",
overwrite=True,
compression=compression,
)
outfile = MDF.concatenate(
[outfile0, outfile1, outfile2, outfile3, outfile4],
version=MDF(input_file).version,
sync=whence,
).save(
Path(TestMDF.tempdir.name) / "tmp_cut",
overwrite=True,
compression=compression,
)
with MDF(outfile) as mdf:
for i, group in enumerate(mdf.groups):
if i == 0:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(vals, v * (j - 1))
if not cond:
print(i, j, vals, v * (j - 1),
len(vals), len(v))
self.assertTrue(cond)
elif i == 1:
v = np.ones(cycles, dtype=np.int64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(
vals,
v * (j - 1) - 0.5)
if not cond:
print(vals,
v * (j - 1) - 0.5, len(vals),
len(v))
self.assertTrue(cond)
elif i == 2:
v = np.arange(cycles, dtype=np.int64) / 100.0
form = "{} * sin(v)"
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
f = form.format(j - 1)
cond = np.array_equal(
vals, numexpr.evaluate(f))
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 3:
v = np.ones(cycles, dtype=np.int64)
form = "({} * v -0.5) / 1"
for j in range(1, 20):
f = form.format(j - 1)
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(
vals, numexpr.evaluate(f))
target = numexpr.evaluate(f)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 4:
for j in range(1, 20):
target = np.array([
"Channel {} sample {}".format(
j, k).encode("ascii")
for k in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 5:
v = np.ones(cycles, dtype=np.dtype("(8,)u1"))
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 6:
for j in range(1, 20):
target = np.array([
b'Value %d' % j for _ in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
input_folder = "input"
output_folder = "output"
# load MDF/DBC files from input folder
path = Path(__file__).parent.absolute()
path_in = Path(path, input_folder)
path_out = Path(path, output_folder)
logfiles = list(path_in.glob("*" + mdf_extension))
dbc = {"CAN": list(path_in.glob("*" + ".DBC"))}
signals = ["EngineSpeed", "WheelBasedVehicleSpeed"]
print("Log file(s): ", logfiles, "\nDBC(s): ", dbc)
# concatenate MDF files from input folder and export as CSV incl. timestamps (localized time)
mdf = MDF.concatenate(logfiles)
mdf.save(Path(path_out, "conc"), overwrite=True)
mdf.export("csv",
filename=Path(path_out, "conc"),
time_as_date=True,
time_from_zero=False)
# extract info from meta header - e.g. to get correct start time of a file
session_start = mdf.header.start_time
delta_seconds = mdf.select(["CAN_DataFrame.BusChannel"])[0].timestamps[0]
split_start = session_start + timedelta(seconds=delta_seconds)
split_start_str = split_start.strftime("%Y%m%d%H%M%S")
# filter an MDF
mdf_filter = mdf.filter(["CAN_DataFrame.ID", "CAN_DataFrame.DataBytes"])
mdf.save(Path(path_out, "filtered"), overwrite=True)
def test_cut_arrays(self):
print("MDF cut big array files")
for mdfname in os.listdir('tmpdir_array'):
for memory in MEMORY:
for whence in (0, 1):
input_file = os.path.join('tmpdir_array', mdfname)
print(input_file, memory, whence)
outfile1 = MDF(input_file, memory=memory)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105.5,
whence=whence).save('tmp1',
overwrite=True)
outfile2 = MDF(input_file, memory=memory)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105.5,
stop=201.5,
whence=whence).save('tmp2',
overwrite=True)
outfile3 = MDF(input_file, memory=memory)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(start=201.5,
whence=whence).save('tmp3',
overwrite=True)
outfile = MDF.concatenate(
[outfile1, outfile2, outfile3],
MDF(input_file, memory='minimum').version,
memory=memory,
).save('tmp_cut', overwrite=True)
equal = True
with MDF(outfile, memory=memory) as mdf:
mdf.configure(read_fragment_size=8000)
for i, group in enumerate(mdf.groups):
if i == 0:
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64),
np.ones((cycles, 2), dtype=np.uint64),
np.ones((cycles, 3), dtype=np.uint64),
]
for j in range(1, 20):
types = [
('Channel_{}'.format(j), '(2, 3)<u8'),
('channel_{}_axis_1'.format(j),
'(2, )<u8'),
('channel_{}_axis_2'.format(j),
'(3, )<u8'),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)[0]
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
print(i,
j,
len(target),
len(vals),
vals,
target,
sep='\n\n')
1 / 0
elif i == 1:
samples = np.ones((cycles, 2, 3),
dtype=np.uint64)
axis_0 = np.ones((cycles, 2), dtype=np.uint64)
axis_1 = np.ones((cycles, 3), dtype=np.uint64)
for j in range(1, 20):
types = [
('Channel_{}'.format(j), '(2, 3)<u8'),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)[0]
target = [
samples * j,
]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
1 / 0
elif i == 2:
samples = [
np.ones(cycles, dtype=np.uint8),
np.ones(cycles, dtype=np.uint16),
np.ones(cycles, dtype=np.uint32),
np.ones(cycles, dtype=np.uint64),
np.ones(cycles, dtype=np.int8),
np.ones(cycles, dtype=np.int16),
np.ones(cycles, dtype=np.int32),
np.ones(cycles, dtype=np.int64),
]
for j in range(1, 20):
types = [
('struct_{}_channel_0'.format(j),
np.uint8),
('struct_{}_channel_1'.format(j),
np.uint16),
('struct_{}_channel_2'.format(j),
np.uint32),
('struct_{}_channel_3'.format(j),
np.uint64),
('struct_{}_channel_4'.format(j),
np.int8),
('struct_{}_channel_5'.format(j),
np.int16),
('struct_{}_channel_6'.format(j),
np.int32),
('struct_{}_channel_7'.format(j),
np.int64),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)[0]
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
1 / 0
self.assertTrue(equal)
cleanup_files()
def test_cut(self):
print("MDF cut big files tests")
t = np.arange(cycles, dtype=np.float64)
for mdfname in os.listdir('tmpdir'):
for memory in MEMORY:
input_file = os.path.join('tmpdir', mdfname)
for whence in (0, 1):
print(input_file, memory)
outfile0 = MDF(input_file, memory=memory)
outfile0.configure(read_fragment_size=8000)
outfile0 = outfile0.cut(stop=-1,
whence=whence).save('tmp0',
overwrite=True)
outfile1 = MDF(input_file, memory=memory)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105,
whence=whence).save('tmp1',
overwrite=True)
outfile2 = MDF(input_file, memory=memory)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105.1,
stop=201,
whence=whence).save('tmp2',
overwrite=True)
outfile3 = MDF(input_file, memory=memory)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(start=201.1,
whence=whence).save('tmp3',
overwrite=True)
outfile4 = MDF(input_file, memory=memory)
outfile4.configure(read_fragment_size=8000)
outfile4 = outfile4.cut(start=7000,
whence=whence).save('tmp4',
overwrite=True)
outfile = MDF.concatenate(
[outfile0, outfile1, outfile2, outfile3, outfile4],
version=MDF(input_file, memory='minimum').version,
memory=memory,
).save('tmp_cut', overwrite=True)
with MDF(outfile) as mdf:
for i, group in enumerate(mdf.groups):
if i == 0:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(vals, v * (j - 1))
if not cond:
print(i, j, vals, v * (j - 1),
len(vals), len(v))
self.assertTrue(cond)
elif i == 1:
v = np.ones(cycles, dtype=np.int64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(
vals,
v * (j - 1) - 0.5)
if not cond:
print(vals,
v * (j - 1) - 0.5, len(vals),
len(v))
self.assertTrue(cond)
elif i == 2:
v = np.arange(cycles, dtype=np.int64) / 100.0
form = '{} * sin(v)'
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
f = form.format(j - 1)
cond = np.array_equal(
vals, numexpr.evaluate(f))
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 3:
v = np.ones(cycles, dtype=np.int64)
form = '({} * v -0.5) / 1'
for j in range(1, 20):
f = form.format(j - 1)
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(
vals, numexpr.evaluate(f))
target = numexpr.evaluate(f)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 4:
for j in range(1, 20):
target = np.array([
'Channel {} sample {}'.format(
j, k).encode('ascii')
for k in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 5:
v = np.ones(cycles, dtype=np.dtype('(8,)u1'))
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 6:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
cleanup_files()