def cut_asam(output, fmt):
x = MDF(r"test.mf4")
t = x.get_master(0)
start, stop = 0.2 * (t[-1] - t[0]) + t[0], 0.8 * (t[-1] - t[0]) + t[0]
with Timer("Cut file", f"asammdf {asammdf_version} mdfv4", fmt) as timer:
x = x.cut(start=start, stop=stop)
output.send([timer.output, timer.error])
def test_cut_arrays(self):
print("MDF cut big array files")
for input_file in Path(TestMDF.tempdir_array.name).iterdir():
for whence in (0, 1):
print(input_file, whence)
outfile1 = MDF(input_file)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(
stop=105.5, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp1", overwrite=True)
outfile2 = MDF(input_file)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105.5,
stop=201.5,
whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) /
"tmp2",
overwrite=True)
outfile3 = MDF(input_file)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(
start=201.5, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp3", overwrite=True)
outfile = MDF.concatenate(
[outfile1, outfile2, outfile3],
MDF(input_file).version).save(Path(TestMDF.tempdir.name) /
"tmp_cut",
overwrite=True)
equal = True
with MDF(outfile) as mdf:
mdf.configure(read_fragment_size=8000)
for i, group in enumerate(mdf.groups):
if i == 0:
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64),
np.ones((cycles, 2), dtype=np.uint64),
np.ones((cycles, 3), dtype=np.uint64),
]
for j in range(1, 20):
types = [
("Channel_{}".format(j), "(2, 3)<u8"),
("channel_{}_axis_1".format(j),
"(2, )<u8"),
("channel_{}_axis_2".format(j),
"(3, )<u8"),
]
types = np.dtype(types)
vals = mdf.get(
"Channel_{}".format(j),
group=i,
samples_only=True,
)[0]
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
print(
i,
j,
len(target),
len(vals),
vals,
target,
sep="\n\n",
)
1 / 0
elif i == 1:
samples = np.ones((cycles, 2, 3), dtype=np.uint64)
axis_0 = np.ones((cycles, 2), dtype=np.uint64)
axis_1 = np.ones((cycles, 3), dtype=np.uint64)
for j in range(1, 20):
types = [("Channel_{}".format(j), "(2, 3)<u8")]
types = np.dtype(types)
vals = mdf.get(
"Channel_{}".format(j),
group=i,
samples_only=True,
)[0]
target = [samples * j]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
1 / 0
elif i == 2:
samples = [
np.ones(cycles, dtype=np.uint8),
np.ones(cycles, dtype=np.uint16),
np.ones(cycles, dtype=np.uint32),
np.ones(cycles, dtype=np.uint64),
np.ones(cycles, dtype=np.int8),
np.ones(cycles, dtype=np.int16),
np.ones(cycles, dtype=np.int32),
np.ones(cycles, dtype=np.int64),
]
for j in range(1, 20):
types = [
("struct_{}_channel_0".format(j),
np.uint8),
("struct_{}_channel_1".format(j),
np.uint16),
("struct_{}_channel_2".format(j),
np.uint32),
("struct_{}_channel_3".format(j),
np.uint64),
("struct_{}_channel_4".format(j), np.int8),
("struct_{}_channel_5".format(j),
np.int16),
("struct_{}_channel_6".format(j),
np.int32),
("struct_{}_channel_7".format(j),
np.int64),
]
types = np.dtype(types)
vals = mdf.get(
"Channel_{}".format(j),
group=i,
samples_only=True,
)[0]
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
1 / 0
self.assertTrue(equal)
def test_cut(self):
print("MDF cut big files tests")
t = np.arange(cycles, dtype=np.float64)
for input_file in Path(TestMDF.tempdir_general.name).iterdir():
for whence in (0, 1):
for compression in range(3):
print(input_file)
outfile0 = MDF(input_file)
outfile0.configure(read_fragment_size=8000)
outfile0 = outfile0.cut(stop=-1,
whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) /
"tmp0",
overwrite=True,
compression=compression,
)
outfile1 = MDF(input_file)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105,
whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) /
"tmp1",
overwrite=True,
compression=compression,
)
outfile2 = MDF(input_file)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105.1,
stop=201,
whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) /
"tmp2",
overwrite=True,
compression=compression,
)
outfile3 = MDF(input_file)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(
start=201.1, whence=whence, include_ends=False).save(
Path(TestMDF.tempdir.name) / "tmp3",
overwrite=True)
outfile4 = MDF(input_file)
outfile4.configure(read_fragment_size=8000)
outfile4 = outfile4.cut(start=7000,
whence=whence,
include_ends=False).save(
Path(TestMDF.tempdir.name) /
"tmp4",
overwrite=True,
compression=compression,
)
outfile = MDF.concatenate(
[outfile0, outfile1, outfile2, outfile3, outfile4],
version=MDF(input_file).version,
sync=whence,
).save(
Path(TestMDF.tempdir.name) / "tmp_cut",
overwrite=True,
compression=compression,
)
with MDF(outfile) as mdf:
for i, group in enumerate(mdf.groups):
if i == 0:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(vals, v * (j - 1))
if not cond:
print(i, j, vals, v * (j - 1),
len(vals), len(v))
self.assertTrue(cond)
elif i == 1:
v = np.ones(cycles, dtype=np.int64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(
vals,
v * (j - 1) - 0.5)
if not cond:
print(vals,
v * (j - 1) - 0.5, len(vals),
len(v))
self.assertTrue(cond)
elif i == 2:
v = np.arange(cycles, dtype=np.int64) / 100.0
form = "{} * sin(v)"
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
f = form.format(j - 1)
cond = np.array_equal(
vals, numexpr.evaluate(f))
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 3:
v = np.ones(cycles, dtype=np.int64)
form = "({} * v -0.5) / 1"
for j in range(1, 20):
f = form.format(j - 1)
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(
vals, numexpr.evaluate(f))
target = numexpr.evaluate(f)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 4:
for j in range(1, 20):
target = np.array([
"Channel {} sample {}".format(
j, k).encode("ascii")
for k in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 5:
v = np.ones(cycles, dtype=np.dtype("(8,)u1"))
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 6:
for j in range(1, 20):
target = np.array([
b'Value %d' % j for _ in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
# create empty MDf version 4.00 file
mdf4 = MDF(version='4.10')
# append the 3 signals to the new file
signals = [s_uint8, s_int32, s_float64]
mdf4.append(signals, 'Created by Python')
# save new file
mdf4.save('my_new_file.mf4', overwrite=True)
# convert new file to mdf version 3.10 with lowest possible RAM usage
mdf3 = mdf4.convert('3.10', memory='minimum')
print(mdf3.version)
# get the float signal
sig = mdf3.get('Float64_Signal')
print(sig)
# cut measurement from 0.3s to end of measurement
mdf4_cut = mdf4.cut(start=0.3)
mdf4_cut.get('Float64_Signal').plot()
# cut measurement from start of measurement to 0.4s
mdf4_cut = mdf4.cut(stop=0.45)
mdf4_cut.get('Float64_Signal').plot()
# filter some signals from the file
mdf4 = mdf4.filter(['Int32_Signal', 'Uint8_Signal'])
# save using zipped transpose deflate blocks
mdf4.save('out.mf4', compression=2, overwrite=True)
def test_cut_arrays(self):
print("MDF cut big array files")
for mdfname in os.listdir('tmpdir_array'):
for memory in MEMORY:
for whence in (0, 1):
input_file = os.path.join('tmpdir_array', mdfname)
print(input_file, memory, whence)
outfile1 = MDF(input_file, memory=memory)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105,
whence=whence).save('tmp1',
overwrite=True)
outfile2 = MDF(input_file, memory=memory)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105, stop=201,
whence=whence).save('tmp2',
overwrite=True)
outfile3 = MDF(input_file, memory=memory)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(start=201,
whence=whence).save('tmp3',
overwrite=True)
outfile = MDF.merge(
[outfile1, outfile2, outfile3],
MDF(input_file, memory='minimum').version,
memory=memory,
).save('tmp_cut', overwrite=True)
equal = True
with MDF(input_file, memory=memory) as mdf:
mdf.configure(read_fragment_size=8000)
for i, group in enumerate(mdf.groups):
if i == 0:
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64),
np.ones((cycles, 2), dtype=np.uint64),
np.ones((cycles, 3), dtype=np.uint64),
]
for j in range(1, 20):
types = [
('Channel_{}'.format(j), '(2, 3)<u8'),
('channel_{}_axis_1'.format(j),
'(2, )<u8'),
('channel_{}_axis_2'.format(j),
'(3, )<u8'),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
elif i == 1:
samples = np.ones((cycles, 2, 3),
dtype=np.uint64)
axis_0 = np.ones((cycles, 2), dtype=np.uint64)
axis_1 = np.ones((cycles, 3), dtype=np.uint64)
for j in range(1, 20):
types = [
('Channel_{}'.format(j), '(2, 3)<u8'),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)
target = [
samples * j,
]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
elif i == 2:
samples = [
np.ones(cycles, dtype=np.uint8),
np.ones(cycles, dtype=np.uint16),
np.ones(cycles, dtype=np.uint32),
np.ones(cycles, dtype=np.uint64),
np.ones(cycles, dtype=np.int8),
np.ones(cycles, dtype=np.int16),
np.ones(cycles, dtype=np.int32),
np.ones(cycles, dtype=np.int64),
]
for j in range(1, 20):
types = [
('struct_{}_channel_0'.format(j),
np.uint8),
('struct_{}_channel_1'.format(j),
np.uint16),
('struct_{}_channel_2'.format(j),
np.uint32),
('struct_{}_channel_3'.format(j),
np.uint64),
('struct_{}_channel_4'.format(j),
np.int8),
('struct_{}_channel_5'.format(j),
np.int16),
('struct_{}_channel_6'.format(j),
np.int32),
('struct_{}_channel_7'.format(j),
np.int64),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
self.assertTrue(equal)
cleanup_files()
def test_cut(self):
print("MDF cut big files tests")
t = np.arange(cycles, dtype=np.float64)
for mdfname in os.listdir('tmpdir'):
for memory in MEMORY:
input_file = os.path.join('tmpdir', mdfname)
for whence in (0, 1):
print(input_file, memory)
outfile0 = MDF(input_file, memory=memory)
outfile0.configure(read_fragment_size=8000)
outfile0 = outfile0.cut(stop=-1,
whence=whence).save('tmp0',
overwrite=True)
outfile1 = MDF(input_file, memory=memory)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105,
whence=whence).save('tmp1',
overwrite=True)
outfile2 = MDF(input_file, memory=memory)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105.1,
stop=201,
whence=whence).save('tmp2',
overwrite=True)
outfile3 = MDF(input_file, memory=memory)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(start=201.1,
whence=whence).save('tmp3',
overwrite=True)
outfile4 = MDF(input_file, memory=memory)
outfile4.configure(read_fragment_size=8000)
outfile4 = outfile4.cut(start=7000,
whence=whence).save('tmp4',
overwrite=True)
outfile = MDF.merge(
[outfile0, outfile1, outfile2, outfile3, outfile4],
MDF(input_file, memory='minimum').version,
memory=memory,
).save('tmp_cut', overwrite=True)
with MDF(outfile) as mdf:
for i, group in enumerate(mdf.groups):
if i == 0:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)
cond = np.array_equal(vals, v * (j - 1))
if not cond:
print(i, j, vals, v * (j - 1),
len(vals), len(v))
self.assertTrue(cond)
elif i == 1:
v = np.ones(cycles, dtype=np.int64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)
cond = np.array_equal(
vals,
v * (j - 1) - 0.5)
if not cond:
print(vals,
v * (j - 1) - 0.5, len(vals),
len(v))
self.assertTrue(cond)
elif i == 2:
v = np.arange(cycles, dtype=np.int64) / 100.0
form = '{} * sin(v)'
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)
f = form.format(j - 1)
cond = np.array_equal(
vals, numexpr.evaluate(f))
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 3:
v = np.ones(cycles, dtype=np.int64)
form = '({} * v -0.5) / 1'
for j in range(1, 20):
f = form.format(j - 1)
vals = mdf.get(group=i,
index=j,
samples_only=True)
cond = np.array_equal(
vals, numexpr.evaluate(f))
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 4:
for j in range(1, 20):
target = np.array([
'Channel {} sample {}'.format(
j, k).encode('ascii')
for k in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 5:
v = np.ones(cycles, dtype=np.dtype('(8,)u1'))
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 6:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
cleanup_files()
